diff --git a/optimized/.DS_Store b/optimized/.DS_Store index 0b351e4d91168d56bd3efe996a0b199f1c0fe325..87d168b65a5132feba66fa3b9646bf757f658891 100644 Binary files a/optimized/.DS_Store and b/optimized/.DS_Store differ diff --git a/optimized/AMI/.DS_Store b/optimized/AMI/.DS_Store index d38edc202947f44e1caa37c69f418d425956aaad..4eec0e4655e57190e556c5d003a659e657c77e15 100644 Binary files a/optimized/AMI/.DS_Store and b/optimized/AMI/.DS_Store differ diff --git a/optimized/AMI/ls_eend_ami_100ms.mlmodelc/analytics/coremldata.bin b/optimized/AMI/ls_eend_ami_100ms.mlmodelc/analytics/coremldata.bin index f6f852da7b64b0c07d874c5c5676ba7d13b4702d..d4e82bfb149adad1fdbfafae79bb9cb328a9b24d 100644 --- a/optimized/AMI/ls_eend_ami_100ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/AMI/ls_eend_ami_100ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c99b87e36d791d93a1545eafd5eeb0ad861ef7f3fce072ddf69728552e1b4a9d +oid sha256:28f8da223ea9be3edf3db973ae344070b02d03c33df4b941525087f093c49abf size 243 diff --git a/optimized/AMI/ls_eend_ami_100ms.mlmodelc/coremldata.bin b/optimized/AMI/ls_eend_ami_100ms.mlmodelc/coremldata.bin index 335f048218e743d5ceac1eddca4d5c48129d6589..669acfcd833bb2d74fd11f4014fc1aac52e093e8 100644 --- a/optimized/AMI/ls_eend_ami_100ms.mlmodelc/coremldata.bin +++ b/optimized/AMI/ls_eend_ami_100ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fe772455e0938b6dd7e6946e93b1f718d6f34193fec92549176b3ec42b0c1e1e -size 1250 +oid sha256:de22c7ff592e38746d6d23bbee0c95372f660224216c047992aca88120ba0865 +size 1341 diff --git a/optimized/AMI/ls_eend_ami_100ms.mlmodelc/metadata.json b/optimized/AMI/ls_eend_ami_100ms.mlmodelc/metadata.json index 6e49dc93ba1199ad3f45e03c7125f7cfab1c7dc8..f8367a87dc29c2dafec58a688955d0a01a37466d 100644 --- a/optimized/AMI/ls_eend_ami_100ms.mlmodelc/metadata.json +++ b/optimized/AMI/ls_eend_ami_100ms.mlmodelc/metadata.json @@ -96,7 +96,7 @@ "Ios17.reshape" : 74, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 53, + "Ios17.transpose" : 57, "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 58, @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"ami\", \"model_label\": \"AMI\", \"variant\": \"pipeline\", \"T\": 1, \"max_speakers\": 4, \"max_nspks\": 6, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"ami\", \"model_label\": \"AMI\", \"variant\": \"pipeline\", \"chunk_size\": 1, \"step_duration_ms\": 100, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 4, \"max_nspks\": 6, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/AMI/ls_eend_ami_100ms.mlmodelc/model.mil b/optimized/AMI/ls_eend_ami_100ms.mlmodelc/model.mil index 6002f61627561041e79bd1e2338259cfc6eaafca..4b17a362e381d6f1a23394343fc7247df457b040 100644 --- a/optimized/AMI/ls_eend_ami_100ms.mlmodelc/model.mil +++ b/optimized/AMI/ls_eend_ami_100ms.mlmodelc/model.mil @@ -216,6 +216,7 @@ program(1.0) tensor var_21 = const()[name = tensor("op_21"), val = tensor(0x0p+0)]; tensor var_24 = const()[name = tensor("op_24"), val = tensor(0x1.0c6f7ap-20)]; tensor var_27 = const()[name = tensor("op_27"), val = tensor(2)]; + tensor var_30 = const()[name = tensor("op_30"), val = tensor(0)]; tensor var_33 = const()[name = tensor("op_33"), val = tensor(1)]; tensor var_35 = const()[name = tensor("op_35"), val = tensor(0x1.4f8b58p-17)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(true)]; @@ -262,8 +263,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_53")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_54")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_196 = const()[name = tensor("op_196"), val = tensor([1, 1])]; tensor var_197 = reshape(shape = var_196, x = sqrt_s_t_1)[name = tensor("op_197")]; @@ -271,7 +272,7 @@ program(1.0) tensor var_199 = mul(x = qk_1, y = M_1)[name = tensor("op_199")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_52")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_199, y = v_1)[name = tensor("inner_1")]; tensor var_201_transpose_x_0 = const()[name = tensor("op_201_transpose_x_0"), val = tensor(false)]; tensor var_201_transpose_y_0 = const()[name = tensor("op_201_transpose_y_0"), val = tensor(false)]; @@ -292,7 +293,7 @@ program(1.0) tensor var_214 = real_div(x = new_kv_unnorm_1, y = var_213)[name = tensor("op_214")]; tensor var_215_perm_0 = const()[name = tensor("op_215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_51")]; + tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_24, x = var_215)[name = tensor("out_3")]; tensor var_219 = const()[name = tensor("op_219"), val = tensor([1, 1, 256])]; tensor out_5 = reshape(shape = var_219, x = out_3)[name = tensor("out_5")]; @@ -336,10 +337,12 @@ program(1.0) tensor var_252 = sub(x = var_17, y = use_roll_1)[name = tensor("op_252")]; tensor var_253 = mul(x = filled_window_1, y = var_252)[name = tensor("op_253")]; tensor var_254 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_254")]; - tensor input_21 = add(x = var_253, y = var_254)[name = tensor("input_21")]; + tensor window_3 = add(x = var_253, y = var_254)[name = tensor("window_3")]; tensor var_256 = const()[name = tensor("op_256"), val = tensor(0x1p+0)]; tensor var_257 = add(x = filled_1, y = var_256)[name = tensor("op_257")]; tensor clip_1 = clip(alpha = var_21, beta = var_16, x = var_257)[name = tensor("clip_1")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_30, interleave = input_21_interleave_0, values = window_3)[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; @@ -348,13 +351,13 @@ program(1.0) tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_50")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_278_split_sizes_0 = const()[name = tensor("op_278_split_sizes_0"), val = tensor([256, 256])]; - tensor var_278_axis_0 = const()[name = tensor("op_278_axis_0"), val = tensor(1)]; - tensor var_278_0, tensor var_278_1 = split(axis = var_278_axis_0, split_sizes = var_278_split_sizes_0, x = inputs_3)[name = tensor("op_278")]; - tensor var_280 = sigmoid(x = var_278_1)[name = tensor("op_280")]; - tensor inputs_5 = mul(x = var_278_0, y = var_280)[name = tensor("inputs_5")]; + tensor var_280_split_sizes_0 = const()[name = tensor("op_280_split_sizes_0"), val = tensor([256, 256])]; + tensor var_280_axis_0 = const()[name = tensor("op_280_axis_0"), val = tensor(1)]; + tensor var_280_0, tensor var_280_1 = split(axis = var_280_axis_0, split_sizes = var_280_split_sizes_0, x = inputs_3)[name = tensor("op_280")]; + tensor var_282 = sigmoid(x = var_280_1)[name = tensor("op_282")]; + tensor inputs_5 = mul(x = var_280_0, y = var_282)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; @@ -374,22 +377,22 @@ program(1.0) tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_311_begin_0 = const()[name = tensor("op_311_begin_0"), val = tensor([0, -1, 0])]; - tensor var_311_end_0 = const()[name = tensor("op_311_end_0"), val = tensor([1, 16, 256])]; - tensor var_311_end_mask_0 = const()[name = tensor("op_311_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_49")]; - tensor var_311 = slice_by_index(begin = var_311_begin_0, end = var_311_end_0, end_mask = var_311_end_mask_0, x = conv_out_1)[name = tensor("op_311")]; - tensor var_314_interleave_0 = const()[name = tensor("op_314_interleave_0"), val = tensor(false)]; - tensor var_314 = concat(axis = var_33, interleave = var_314_interleave_0, values = var_311)[name = tensor("op_314")]; - tensor input_31 = add(x = x_3, y = var_314)[name = tensor("input_31")]; + tensor var_313_begin_0 = const()[name = tensor("op_313_begin_0"), val = tensor([0, -1, 0])]; + tensor var_313_end_0 = const()[name = tensor("op_313_end_0"), val = tensor([1, 16, 256])]; + tensor var_313_end_mask_0 = const()[name = tensor("op_313_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_313 = slice_by_index(begin = var_313_begin_0, end = var_313_end_0, end_mask = var_313_end_mask_0, x = conv_out_1)[name = tensor("op_313")]; + tensor var_315_perm_0 = const()[name = tensor("op_315_perm_0"), val = tensor([1, 0, 2])]; + tensor var_315 = transpose(perm = var_315_perm_0, x = var_313)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_315)[name = tensor("input_31")]; tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; - tensor var_337 = const()[name = tensor("op_337"), val = tensor(0x1p-1)]; - tensor var_338 = mul(x = input_39, y = var_337)[name = tensor("op_338")]; - tensor input_41 = add(x = var_338, y = input_31)[name = tensor("input_41")]; + tensor var_338 = const()[name = tensor("op_338"), val = tensor(0x1p-1)]; + tensor var_339 = mul(x = input_39, y = var_338)[name = tensor("op_339")]; + tensor input_41 = add(x = var_339, y = input_31)[name = tensor("input_41")]; tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; @@ -397,9 +400,9 @@ program(1.0) tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; - tensor var_367 = const()[name = tensor("op_367"), val = tensor(0x1p-1)]; - tensor var_368 = mul(x = input_51, y = var_367)[name = tensor("op_368")]; - tensor input_53 = add(x = var_368, y = input_43)[name = tensor("input_53")]; + tensor var_368 = const()[name = tensor("op_368"), val = tensor(0x1p-1)]; + tensor var_369 = mul(x = input_51, y = var_368)[name = tensor("op_369")]; + tensor input_53 = add(x = var_369, y = input_43)[name = tensor("input_53")]; tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; @@ -412,19 +415,19 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_382 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; - tensor var_383 = const()[name = tensor("op_383"), val = tensor([1, 1, 4, 64])]; - tensor var_384 = reshape(shape = var_383, x = var_382)[name = tensor("op_384")]; + tensor var_383 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_384 = const()[name = tensor("op_384"), val = tensor([1, 1, 4, 64])]; + tensor var_385 = reshape(shape = var_384, x = var_383)[name = tensor("op_385")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_388 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; - tensor var_389 = const()[name = tensor("op_389"), val = tensor(0x1p-3)]; - tensor var_390 = mul(x = var_388, y = var_389)[name = tensor("op_390")]; - tensor var_391 = const()[name = tensor("op_391"), val = tensor([1, 1, 4, 64])]; - tensor var_392 = reshape(shape = var_391, x = var_390)[name = tensor("op_392")]; + tensor var_389 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_390 = const()[name = tensor("op_390"), val = tensor(0x1p-3)]; + tensor var_391 = mul(x = var_389, y = var_390)[name = tensor("op_391")]; + tensor var_392 = const()[name = tensor("op_392"), val = tensor([1, 1, 4, 64])]; + tensor var_393 = reshape(shape = var_392, x = var_391)[name = tensor("op_393")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_396 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; - tensor var_397 = const()[name = tensor("op_397"), val = tensor([1, 1, 4, 64])]; - tensor var_398 = reshape(shape = var_397, x = var_396)[name = tensor("op_398")]; + tensor var_397 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_398 = const()[name = tensor("op_398"), val = tensor([1, 1, 4, 64])]; + tensor var_399 = reshape(shape = var_398, x = var_397)[name = tensor("op_399")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; @@ -432,84 +435,86 @@ program(1.0) tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_392)[name = tensor("transpose_47")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_384)[name = tensor("transpose_48")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_393)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_385)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_408 = const()[name = tensor("op_408"), val = tensor([1, 1])]; - tensor var_409 = reshape(shape = var_408, x = sqrt_s_t_3)[name = tensor("op_409")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_409)[name = tensor("M_3")]; - tensor var_411 = mul(x = qk_3, y = M_3)[name = tensor("op_411")]; + tensor var_409 = const()[name = tensor("op_409"), val = tensor([1, 1])]; + tensor var_410 = reshape(shape = var_409, x = sqrt_s_t_3)[name = tensor("op_410")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_410)[name = tensor("M_3")]; + tensor var_412 = mul(x = qk_3, y = M_3)[name = tensor("op_412")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_398)[name = tensor("transpose_46")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_411, y = v_3)[name = tensor("inner_3")]; - tensor var_413_transpose_x_0 = const()[name = tensor("op_413_transpose_x_0"), val = tensor(false)]; - tensor var_413_transpose_y_0 = const()[name = tensor("op_413_transpose_y_0"), val = tensor(false)]; - tensor var_413 = matmul(transpose_x = var_413_transpose_x_0, transpose_y = var_413_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_413")]; - tensor var_414 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_414")]; - tensor var_415 = const()[name = tensor("op_415"), val = tensor([1, 1, 1, 1])]; - tensor var_416 = reshape(shape = var_415, x = var_414)[name = tensor("op_416")]; - tensor cross_3 = mul(x = var_413, y = var_416)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_399)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_412, y = v_3)[name = tensor("inner_3")]; + tensor var_414_transpose_x_0 = const()[name = tensor("op_414_transpose_x_0"), val = tensor(false)]; + tensor var_414_transpose_y_0 = const()[name = tensor("op_414_transpose_y_0"), val = tensor(false)]; + tensor var_414 = matmul(transpose_x = var_414_transpose_x_0, transpose_y = var_414_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_414")]; + tensor var_415 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_415")]; + tensor var_416 = const()[name = tensor("op_416"), val = tensor([1, 1, 1, 1])]; + tensor var_417 = reshape(shape = var_416, x = var_415)[name = tensor("op_417")]; + tensor cross_3 = mul(x = var_414, y = var_417)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_419 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_419")]; - tensor var_421_transpose_x_1 = const()[name = tensor("op_421_transpose_x_1"), val = tensor(true)]; - tensor var_421_transpose_y_1 = const()[name = tensor("op_421_transpose_y_1"), val = tensor(false)]; - tensor var_421 = matmul(transpose_x = var_421_transpose_x_1, transpose_y = var_421_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_421")]; - tensor new_kv_unnorm_3 = add(x = var_419, y = var_421)[name = tensor("new_kv_unnorm_3")]; - tensor var_423 = const()[name = tensor("op_423"), val = tensor(0x1p+0)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_423)[name = tensor("new_scale_3")]; - tensor var_425 = sqrt(x = new_scale_3)[name = tensor("op_425")]; - tensor var_426 = real_div(x = new_kv_unnorm_3, y = var_425)[name = tensor("op_426")]; - tensor var_427_perm_0 = const()[name = tensor("op_427_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_420 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_420")]; + tensor var_422_transpose_x_1 = const()[name = tensor("op_422_transpose_x_1"), val = tensor(true)]; + tensor var_422_transpose_y_1 = const()[name = tensor("op_422_transpose_y_1"), val = tensor(false)]; + tensor var_422 = matmul(transpose_x = var_422_transpose_x_1, transpose_y = var_422_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_422")]; + tensor new_kv_unnorm_3 = add(x = var_420, y = var_422)[name = tensor("new_kv_unnorm_3")]; + tensor var_424 = const()[name = tensor("op_424"), val = tensor(0x1p+0)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_424)[name = tensor("new_scale_3")]; + tensor var_426 = sqrt(x = new_scale_3)[name = tensor("op_426")]; + tensor var_427 = real_div(x = new_kv_unnorm_3, y = var_426)[name = tensor("op_427")]; + tensor var_428_perm_0 = const()[name = tensor("op_428_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_427 = transpose(perm = var_427_perm_0, x = out_7)[name = tensor("transpose_45")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_427)[name = tensor("out_9")]; - tensor var_431 = const()[name = tensor("op_431"), val = tensor([1, 1, 256])]; - tensor out_11 = reshape(shape = var_431, x = out_9)[name = tensor("out_11")]; - tensor var_433 = silu(x = input_57)[name = tensor("op_433")]; - tensor input_59 = mul(x = var_433, y = out_11)[name = tensor("input_59")]; + tensor var_428 = transpose(perm = var_428_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_428)[name = tensor("out_9")]; + tensor var_432 = const()[name = tensor("op_432"), val = tensor([1, 1, 256])]; + tensor out_11 = reshape(shape = var_432, x = out_9)[name = tensor("out_11")]; + tensor var_434 = silu(x = input_57)[name = tensor("op_434")]; + tensor input_59 = mul(x = var_434, y = out_11)[name = tensor("input_59")]; tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; tensor filled_3_begin_0 = const()[name = tensor("filled_3_begin_0"), val = tensor([1, 0])]; tensor filled_3_end_0 = const()[name = tensor("filled_3_end_0"), val = tensor([2, 1])]; tensor filled_3_end_mask_0 = const()[name = tensor("filled_3_end_mask_0"), val = tensor([false, true])]; tensor filled_3_squeeze_mask_0 = const()[name = tensor("filled_3_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_3 = slice_by_index(begin = filled_3_begin_0, end = filled_3_end_0, end_mask = filled_3_end_mask_0, squeeze_mask = filled_3_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_3")]; - tensor var_445_begin_0 = const()[name = tensor("op_445_begin_0"), val = tensor([0, 1, 0])]; - tensor var_445_end_0 = const()[name = tensor("op_445_end_0"), val = tensor([1, 16, 256])]; - tensor var_445_end_mask_0 = const()[name = tensor("op_445_end_mask_0"), val = tensor([true, true, true])]; - tensor var_445 = slice_by_index(begin = var_445_begin_0, end = var_445_end_0, end_mask = var_445_end_mask_0, x = window_3)[name = tensor("op_445")]; + tensor var_446_begin_0 = const()[name = tensor("op_446_begin_0"), val = tensor([0, 1, 0])]; + tensor var_446_end_0 = const()[name = tensor("op_446_end_0"), val = tensor([1, 16, 256])]; + tensor var_446_end_mask_0 = const()[name = tensor("op_446_end_mask_0"), val = tensor([true, true, true])]; + tensor var_446 = slice_by_index(begin = var_446_begin_0, end = var_446_end_0, end_mask = var_446_end_mask_0, x = window_5)[name = tensor("op_446")]; tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_445, x_9))[name = tensor("rolled_3")]; + tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_446, x_9))[name = tensor("rolled_3")]; tensor clip_2 = clip(alpha = var_21, beta = var_20, x = filled_3)[name = tensor("clip_2")]; - tensor var_450 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_450")]; - tensor var_451_dtype_0 = const()[name = tensor("op_451_dtype_0"), val = tensor("fp32")]; - tensor var_452 = const()[name = tensor("op_452"), val = tensor([1, 16, 1])]; - tensor var_451 = cast(dtype = var_451_dtype_0, x = var_450)[name = tensor("cast_125")]; - tensor fill_mask_3 = reshape(shape = var_452, x = var_451)[name = tensor("fill_mask_3")]; + tensor var_451 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_451")]; + tensor var_452_dtype_0 = const()[name = tensor("op_452_dtype_0"), val = tensor("fp32")]; + tensor var_453 = const()[name = tensor("op_453"), val = tensor([1, 16, 1])]; + tensor var_452 = cast(dtype = var_452_dtype_0, x = var_451)[name = tensor("cast_125")]; + tensor fill_mask_3 = reshape(shape = var_453, x = var_452)[name = tensor("fill_mask_3")]; tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = x_9)[name = tensor("x_expanded_3")]; - tensor var_456 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_456")]; - tensor var_457 = mul(x = window_3, y = var_456)[name = tensor("op_457")]; - tensor var_458 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_458")]; - tensor filled_window_3 = add(x = var_457, y = var_458)[name = tensor("filled_window_3")]; - tensor var_460 = greater_equal(x = filled_3, y = var_16)[name = tensor("op_460")]; - tensor var_461_dtype_0 = const()[name = tensor("op_461_dtype_0"), val = tensor("fp32")]; - tensor var_462 = const()[name = tensor("op_462"), val = tensor([1, 1, 1])]; - tensor var_461 = cast(dtype = var_461_dtype_0, x = var_460)[name = tensor("cast_124")]; - tensor use_roll_3 = reshape(shape = var_462, x = var_461)[name = tensor("use_roll_3")]; - tensor var_464 = sub(x = var_17, y = use_roll_3)[name = tensor("op_464")]; - tensor var_465 = mul(x = filled_window_3, y = var_464)[name = tensor("op_465")]; - tensor var_466 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_466")]; - tensor input_61 = add(x = var_465, y = var_466)[name = tensor("input_61")]; - tensor var_468 = const()[name = tensor("op_468"), val = tensor(0x1p+0)]; - tensor var_469 = add(x = filled_3, y = var_468)[name = tensor("op_469")]; - tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_469)[name = tensor("clip_3")]; + tensor var_457 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_457")]; + tensor var_458 = mul(x = window_5, y = var_457)[name = tensor("op_458")]; + tensor var_459 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_459")]; + tensor filled_window_3 = add(x = var_458, y = var_459)[name = tensor("filled_window_3")]; + tensor var_461 = greater_equal(x = filled_3, y = var_16)[name = tensor("op_461")]; + tensor var_462_dtype_0 = const()[name = tensor("op_462_dtype_0"), val = tensor("fp32")]; + tensor var_463 = const()[name = tensor("op_463"), val = tensor([1, 1, 1])]; + tensor var_462 = cast(dtype = var_462_dtype_0, x = var_461)[name = tensor("cast_124")]; + tensor use_roll_3 = reshape(shape = var_463, x = var_462)[name = tensor("use_roll_3")]; + tensor var_465 = sub(x = var_17, y = use_roll_3)[name = tensor("op_465")]; + tensor var_466 = mul(x = filled_window_3, y = var_465)[name = tensor("op_466")]; + tensor var_467 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_467")]; + tensor window_7 = add(x = var_466, y = var_467)[name = tensor("window_7")]; + tensor var_469 = const()[name = tensor("op_469"), val = tensor(0x1p+0)]; + tensor var_470 = add(x = filled_3, y = var_469)[name = tensor("op_470")]; + tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_470)[name = tensor("clip_3")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_30, interleave = input_61_interleave_0, values = window_7)[name = tensor("input_61")]; tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; @@ -518,13 +523,13 @@ program(1.0) tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; - tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_44")]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; - tensor var_490_split_sizes_0 = const()[name = tensor("op_490_split_sizes_0"), val = tensor([256, 256])]; - tensor var_490_axis_0 = const()[name = tensor("op_490_axis_0"), val = tensor(1)]; - tensor var_490_0, tensor var_490_1 = split(axis = var_490_axis_0, split_sizes = var_490_split_sizes_0, x = inputs_13)[name = tensor("op_490")]; - tensor var_492 = sigmoid(x = var_490_1)[name = tensor("op_492")]; - tensor inputs_15 = mul(x = var_490_0, y = var_492)[name = tensor("inputs_15")]; + tensor var_493_split_sizes_0 = const()[name = tensor("op_493_split_sizes_0"), val = tensor([256, 256])]; + tensor var_493_axis_0 = const()[name = tensor("op_493_axis_0"), val = tensor(1)]; + tensor var_493_0, tensor var_493_1 = split(axis = var_493_axis_0, split_sizes = var_493_split_sizes_0, x = inputs_13)[name = tensor("op_493")]; + tensor var_495 = sigmoid(x = var_493_1)[name = tensor("op_495")]; + tensor inputs_15 = mul(x = var_493_0, y = var_495)[name = tensor("inputs_15")]; tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; @@ -544,22 +549,22 @@ program(1.0) tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_523_begin_0 = const()[name = tensor("op_523_begin_0"), val = tensor([0, -1, 0])]; - tensor var_523_end_0 = const()[name = tensor("op_523_end_0"), val = tensor([1, 16, 256])]; - tensor var_523_end_mask_0 = const()[name = tensor("op_523_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_43")]; - tensor var_523 = slice_by_index(begin = var_523_begin_0, end = var_523_end_0, end_mask = var_523_end_mask_0, x = conv_out_3)[name = tensor("op_523")]; - tensor var_526_interleave_0 = const()[name = tensor("op_526_interleave_0"), val = tensor(false)]; - tensor var_526 = concat(axis = var_33, interleave = var_526_interleave_0, values = var_523)[name = tensor("op_526")]; - tensor input_71 = add(x = x_9, y = var_526)[name = tensor("input_71")]; + tensor var_526_begin_0 = const()[name = tensor("op_526_begin_0"), val = tensor([0, -1, 0])]; + tensor var_526_end_0 = const()[name = tensor("op_526_end_0"), val = tensor([1, 16, 256])]; + tensor var_526_end_mask_0 = const()[name = tensor("op_526_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_526 = slice_by_index(begin = var_526_begin_0, end = var_526_end_0, end_mask = var_526_end_mask_0, x = conv_out_3)[name = tensor("op_526")]; + tensor var_528_perm_0 = const()[name = tensor("op_528_perm_0"), val = tensor([1, 0, 2])]; + tensor var_528 = transpose(perm = var_528_perm_0, x = var_526)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_528)[name = tensor("input_71")]; tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; - tensor var_549 = const()[name = tensor("op_549"), val = tensor(0x1p-1)]; - tensor var_550 = mul(x = input_79, y = var_549)[name = tensor("op_550")]; - tensor input_81 = add(x = var_550, y = input_71)[name = tensor("input_81")]; + tensor var_551 = const()[name = tensor("op_551"), val = tensor(0x1p-1)]; + tensor var_552 = mul(x = input_79, y = var_551)[name = tensor("op_552")]; + tensor input_81 = add(x = var_552, y = input_71)[name = tensor("input_81")]; tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; @@ -567,9 +572,9 @@ program(1.0) tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; - tensor var_579 = const()[name = tensor("op_579"), val = tensor(0x1p-1)]; - tensor var_580 = mul(x = input_91, y = var_579)[name = tensor("op_580")]; - tensor input_93 = add(x = var_580, y = input_83)[name = tensor("input_93")]; + tensor var_581 = const()[name = tensor("op_581"), val = tensor(0x1p-1)]; + tensor var_582 = mul(x = input_91, y = var_581)[name = tensor("op_582")]; + tensor input_93 = add(x = var_582, y = input_83)[name = tensor("input_93")]; tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; @@ -582,19 +587,19 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_594 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; - tensor var_595 = const()[name = tensor("op_595"), val = tensor([1, 1, 4, 64])]; - tensor var_596 = reshape(shape = var_595, x = var_594)[name = tensor("op_596")]; + tensor var_596 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_597 = const()[name = tensor("op_597"), val = tensor([1, 1, 4, 64])]; + tensor var_598 = reshape(shape = var_597, x = var_596)[name = tensor("op_598")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_600 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; - tensor var_601 = const()[name = tensor("op_601"), val = tensor(0x1p-3)]; - tensor var_602 = mul(x = var_600, y = var_601)[name = tensor("op_602")]; - tensor var_603 = const()[name = tensor("op_603"), val = tensor([1, 1, 4, 64])]; - tensor var_604 = reshape(shape = var_603, x = var_602)[name = tensor("op_604")]; + tensor var_602 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_603 = const()[name = tensor("op_603"), val = tensor(0x1p-3)]; + tensor var_604 = mul(x = var_602, y = var_603)[name = tensor("op_604")]; + tensor var_605 = const()[name = tensor("op_605"), val = tensor([1, 1, 4, 64])]; + tensor var_606 = reshape(shape = var_605, x = var_604)[name = tensor("op_606")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_608 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; - tensor var_609 = const()[name = tensor("op_609"), val = tensor([1, 1, 4, 64])]; - tensor var_610 = reshape(shape = var_609, x = var_608)[name = tensor("op_610")]; + tensor var_610 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_611 = const()[name = tensor("op_611"), val = tensor([1, 1, 4, 64])]; + tensor var_612 = reshape(shape = var_611, x = var_610)[name = tensor("op_612")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; @@ -602,84 +607,86 @@ program(1.0) tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_604)[name = tensor("transpose_41")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_596)[name = tensor("transpose_42")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_606)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_598)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_620 = const()[name = tensor("op_620"), val = tensor([1, 1])]; - tensor var_621 = reshape(shape = var_620, x = sqrt_s_t_5)[name = tensor("op_621")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_621)[name = tensor("M_5")]; - tensor var_623 = mul(x = qk_5, y = M_5)[name = tensor("op_623")]; + tensor var_622 = const()[name = tensor("op_622"), val = tensor([1, 1])]; + tensor var_623 = reshape(shape = var_622, x = sqrt_s_t_5)[name = tensor("op_623")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_623)[name = tensor("M_5")]; + tensor var_625 = mul(x = qk_5, y = M_5)[name = tensor("op_625")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_610)[name = tensor("transpose_40")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_623, y = v_5)[name = tensor("inner_5")]; - tensor var_625_transpose_x_0 = const()[name = tensor("op_625_transpose_x_0"), val = tensor(false)]; - tensor var_625_transpose_y_0 = const()[name = tensor("op_625_transpose_y_0"), val = tensor(false)]; - tensor var_625 = matmul(transpose_x = var_625_transpose_x_0, transpose_y = var_625_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_625")]; - tensor var_626 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_626")]; - tensor var_627 = const()[name = tensor("op_627"), val = tensor([1, 1, 1, 1])]; - tensor var_628 = reshape(shape = var_627, x = var_626)[name = tensor("op_628")]; - tensor cross_5 = mul(x = var_625, y = var_628)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_612)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_625, y = v_5)[name = tensor("inner_5")]; + tensor var_627_transpose_x_0 = const()[name = tensor("op_627_transpose_x_0"), val = tensor(false)]; + tensor var_627_transpose_y_0 = const()[name = tensor("op_627_transpose_y_0"), val = tensor(false)]; + tensor var_627 = matmul(transpose_x = var_627_transpose_x_0, transpose_y = var_627_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_627")]; + tensor var_628 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_628")]; + tensor var_629 = const()[name = tensor("op_629"), val = tensor([1, 1, 1, 1])]; + tensor var_630 = reshape(shape = var_629, x = var_628)[name = tensor("op_630")]; + tensor cross_5 = mul(x = var_627, y = var_630)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_631 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_631")]; - tensor var_633_transpose_x_1 = const()[name = tensor("op_633_transpose_x_1"), val = tensor(true)]; - tensor var_633_transpose_y_1 = const()[name = tensor("op_633_transpose_y_1"), val = tensor(false)]; - tensor var_633 = matmul(transpose_x = var_633_transpose_x_1, transpose_y = var_633_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_633")]; - tensor new_kv_unnorm_5 = add(x = var_631, y = var_633)[name = tensor("new_kv_unnorm_5")]; - tensor var_635 = const()[name = tensor("op_635"), val = tensor(0x1p+0)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_635)[name = tensor("new_scale_5")]; - tensor var_637 = sqrt(x = new_scale_5)[name = tensor("op_637")]; - tensor var_638 = real_div(x = new_kv_unnorm_5, y = var_637)[name = tensor("op_638")]; - tensor var_639_perm_0 = const()[name = tensor("op_639_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_633 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_633")]; + tensor var_635_transpose_x_1 = const()[name = tensor("op_635_transpose_x_1"), val = tensor(true)]; + tensor var_635_transpose_y_1 = const()[name = tensor("op_635_transpose_y_1"), val = tensor(false)]; + tensor var_635 = matmul(transpose_x = var_635_transpose_x_1, transpose_y = var_635_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_635")]; + tensor new_kv_unnorm_5 = add(x = var_633, y = var_635)[name = tensor("new_kv_unnorm_5")]; + tensor var_637 = const()[name = tensor("op_637"), val = tensor(0x1p+0)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_637)[name = tensor("new_scale_5")]; + tensor var_639 = sqrt(x = new_scale_5)[name = tensor("op_639")]; + tensor var_640 = real_div(x = new_kv_unnorm_5, y = var_639)[name = tensor("op_640")]; + tensor var_641_perm_0 = const()[name = tensor("op_641_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_639 = transpose(perm = var_639_perm_0, x = out_13)[name = tensor("transpose_39")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_639)[name = tensor("out_15")]; - tensor var_643 = const()[name = tensor("op_643"), val = tensor([1, 1, 256])]; - tensor out_17 = reshape(shape = var_643, x = out_15)[name = tensor("out_17")]; - tensor var_645 = silu(x = input_97)[name = tensor("op_645")]; - tensor input_99 = mul(x = var_645, y = out_17)[name = tensor("input_99")]; + tensor var_641 = transpose(perm = var_641_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_641)[name = tensor("out_15")]; + tensor var_645 = const()[name = tensor("op_645"), val = tensor([1, 1, 256])]; + tensor out_17 = reshape(shape = var_645, x = out_15)[name = tensor("out_17")]; + tensor var_647 = silu(x = input_97)[name = tensor("op_647")]; + tensor input_99 = mul(x = var_647, y = out_17)[name = tensor("input_99")]; tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor window_9_begin_0 = const()[name = tensor("window_9_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_9_end_0 = const()[name = tensor("window_9_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_9_end_mask_0 = const()[name = tensor("window_9_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_9_squeeze_mask_0 = const()[name = tensor("window_9_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_9 = slice_by_index(begin = window_9_begin_0, end = window_9_end_0, end_mask = window_9_end_mask_0, squeeze_mask = window_9_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_9")]; tensor filled_5_begin_0 = const()[name = tensor("filled_5_begin_0"), val = tensor([2, 0])]; tensor filled_5_end_0 = const()[name = tensor("filled_5_end_0"), val = tensor([3, 1])]; tensor filled_5_end_mask_0 = const()[name = tensor("filled_5_end_mask_0"), val = tensor([false, true])]; tensor filled_5_squeeze_mask_0 = const()[name = tensor("filled_5_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_5 = slice_by_index(begin = filled_5_begin_0, end = filled_5_end_0, end_mask = filled_5_end_mask_0, squeeze_mask = filled_5_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_5")]; - tensor var_657_begin_0 = const()[name = tensor("op_657_begin_0"), val = tensor([0, 1, 0])]; - tensor var_657_end_0 = const()[name = tensor("op_657_end_0"), val = tensor([1, 16, 256])]; - tensor var_657_end_mask_0 = const()[name = tensor("op_657_end_mask_0"), val = tensor([true, true, true])]; - tensor var_657 = slice_by_index(begin = var_657_begin_0, end = var_657_end_0, end_mask = var_657_end_mask_0, x = window_5)[name = tensor("op_657")]; + tensor var_659_begin_0 = const()[name = tensor("op_659_begin_0"), val = tensor([0, 1, 0])]; + tensor var_659_end_0 = const()[name = tensor("op_659_end_0"), val = tensor([1, 16, 256])]; + tensor var_659_end_mask_0 = const()[name = tensor("op_659_end_mask_0"), val = tensor([true, true, true])]; + tensor var_659 = slice_by_index(begin = var_659_begin_0, end = var_659_end_0, end_mask = var_659_end_mask_0, x = window_9)[name = tensor("op_659")]; tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_657, x_15))[name = tensor("rolled_5")]; + tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_659, x_15))[name = tensor("rolled_5")]; tensor clip_4 = clip(alpha = var_21, beta = var_20, x = filled_5)[name = tensor("clip_4")]; - tensor var_662 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_662")]; - tensor var_663_dtype_0 = const()[name = tensor("op_663_dtype_0"), val = tensor("fp32")]; - tensor var_664 = const()[name = tensor("op_664"), val = tensor([1, 16, 1])]; - tensor var_663 = cast(dtype = var_663_dtype_0, x = var_662)[name = tensor("cast_123")]; - tensor fill_mask_5 = reshape(shape = var_664, x = var_663)[name = tensor("fill_mask_5")]; + tensor var_664 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_664")]; + tensor var_665_dtype_0 = const()[name = tensor("op_665_dtype_0"), val = tensor("fp32")]; + tensor var_666 = const()[name = tensor("op_666"), val = tensor([1, 16, 1])]; + tensor var_665 = cast(dtype = var_665_dtype_0, x = var_664)[name = tensor("cast_123")]; + tensor fill_mask_5 = reshape(shape = var_666, x = var_665)[name = tensor("fill_mask_5")]; tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = x_15)[name = tensor("x_expanded_5")]; - tensor var_668 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_668")]; - tensor var_669 = mul(x = window_5, y = var_668)[name = tensor("op_669")]; - tensor var_670 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_670")]; - tensor filled_window_5 = add(x = var_669, y = var_670)[name = tensor("filled_window_5")]; - tensor var_672 = greater_equal(x = filled_5, y = var_16)[name = tensor("op_672")]; - tensor var_673_dtype_0 = const()[name = tensor("op_673_dtype_0"), val = tensor("fp32")]; - tensor var_674 = const()[name = tensor("op_674"), val = tensor([1, 1, 1])]; - tensor var_673 = cast(dtype = var_673_dtype_0, x = var_672)[name = tensor("cast_122")]; - tensor use_roll_5 = reshape(shape = var_674, x = var_673)[name = tensor("use_roll_5")]; - tensor var_676 = sub(x = var_17, y = use_roll_5)[name = tensor("op_676")]; - tensor var_677 = mul(x = filled_window_5, y = var_676)[name = tensor("op_677")]; - tensor var_678 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_678")]; - tensor input_101 = add(x = var_677, y = var_678)[name = tensor("input_101")]; - tensor var_680 = const()[name = tensor("op_680"), val = tensor(0x1p+0)]; - tensor var_681 = add(x = filled_5, y = var_680)[name = tensor("op_681")]; - tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_681)[name = tensor("clip_5")]; + tensor var_670 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_670")]; + tensor var_671 = mul(x = window_9, y = var_670)[name = tensor("op_671")]; + tensor var_672 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_672")]; + tensor filled_window_5 = add(x = var_671, y = var_672)[name = tensor("filled_window_5")]; + tensor var_674 = greater_equal(x = filled_5, y = var_16)[name = tensor("op_674")]; + tensor var_675_dtype_0 = const()[name = tensor("op_675_dtype_0"), val = tensor("fp32")]; + tensor var_676 = const()[name = tensor("op_676"), val = tensor([1, 1, 1])]; + tensor var_675 = cast(dtype = var_675_dtype_0, x = var_674)[name = tensor("cast_122")]; + tensor use_roll_5 = reshape(shape = var_676, x = var_675)[name = tensor("use_roll_5")]; + tensor var_678 = sub(x = var_17, y = use_roll_5)[name = tensor("op_678")]; + tensor var_679 = mul(x = filled_window_5, y = var_678)[name = tensor("op_679")]; + tensor var_680 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_680")]; + tensor window_11 = add(x = var_679, y = var_680)[name = tensor("window_11")]; + tensor var_682 = const()[name = tensor("op_682"), val = tensor(0x1p+0)]; + tensor var_683 = add(x = filled_5, y = var_682)[name = tensor("op_683")]; + tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_683)[name = tensor("clip_5")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_30, interleave = input_101_interleave_0, values = window_11)[name = tensor("input_101")]; tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; @@ -688,13 +695,13 @@ program(1.0) tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; - tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_38")]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; - tensor var_702_split_sizes_0 = const()[name = tensor("op_702_split_sizes_0"), val = tensor([256, 256])]; - tensor var_702_axis_0 = const()[name = tensor("op_702_axis_0"), val = tensor(1)]; - tensor var_702_0, tensor var_702_1 = split(axis = var_702_axis_0, split_sizes = var_702_split_sizes_0, x = inputs_23)[name = tensor("op_702")]; - tensor var_704 = sigmoid(x = var_702_1)[name = tensor("op_704")]; - tensor inputs_25 = mul(x = var_702_0, y = var_704)[name = tensor("inputs_25")]; + tensor var_706_split_sizes_0 = const()[name = tensor("op_706_split_sizes_0"), val = tensor([256, 256])]; + tensor var_706_axis_0 = const()[name = tensor("op_706_axis_0"), val = tensor(1)]; + tensor var_706_0, tensor var_706_1 = split(axis = var_706_axis_0, split_sizes = var_706_split_sizes_0, x = inputs_23)[name = tensor("op_706")]; + tensor var_708 = sigmoid(x = var_706_1)[name = tensor("op_708")]; + tensor inputs_25 = mul(x = var_706_0, y = var_708)[name = tensor("inputs_25")]; tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; @@ -714,22 +721,22 @@ program(1.0) tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_735_begin_0 = const()[name = tensor("op_735_begin_0"), val = tensor([0, -1, 0])]; - tensor var_735_end_0 = const()[name = tensor("op_735_end_0"), val = tensor([1, 16, 256])]; - tensor var_735_end_mask_0 = const()[name = tensor("op_735_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_37")]; - tensor var_735 = slice_by_index(begin = var_735_begin_0, end = var_735_end_0, end_mask = var_735_end_mask_0, x = conv_out_5)[name = tensor("op_735")]; - tensor var_738_interleave_0 = const()[name = tensor("op_738_interleave_0"), val = tensor(false)]; - tensor var_738 = concat(axis = var_33, interleave = var_738_interleave_0, values = var_735)[name = tensor("op_738")]; - tensor input_111 = add(x = x_15, y = var_738)[name = tensor("input_111")]; + tensor var_739_begin_0 = const()[name = tensor("op_739_begin_0"), val = tensor([0, -1, 0])]; + tensor var_739_end_0 = const()[name = tensor("op_739_end_0"), val = tensor([1, 16, 256])]; + tensor var_739_end_mask_0 = const()[name = tensor("op_739_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_739 = slice_by_index(begin = var_739_begin_0, end = var_739_end_0, end_mask = var_739_end_mask_0, x = conv_out_5)[name = tensor("op_739")]; + tensor var_741_perm_0 = const()[name = tensor("op_741_perm_0"), val = tensor([1, 0, 2])]; + tensor var_741 = transpose(perm = var_741_perm_0, x = var_739)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_741)[name = tensor("input_111")]; tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; - tensor var_761 = const()[name = tensor("op_761"), val = tensor(0x1p-1)]; - tensor var_762 = mul(x = input_119, y = var_761)[name = tensor("op_762")]; - tensor input_121 = add(x = var_762, y = input_111)[name = tensor("input_121")]; + tensor var_764 = const()[name = tensor("op_764"), val = tensor(0x1p-1)]; + tensor var_765 = mul(x = input_119, y = var_764)[name = tensor("op_765")]; + tensor input_121 = add(x = var_765, y = input_111)[name = tensor("input_121")]; tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; @@ -737,9 +744,9 @@ program(1.0) tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; - tensor var_791 = const()[name = tensor("op_791"), val = tensor(0x1p-1)]; - tensor var_792 = mul(x = input_131, y = var_791)[name = tensor("op_792")]; - tensor input_133 = add(x = var_792, y = input_123)[name = tensor("input_133")]; + tensor var_794 = const()[name = tensor("op_794"), val = tensor(0x1p-1)]; + tensor var_795 = mul(x = input_131, y = var_794)[name = tensor("op_795")]; + tensor input_133 = add(x = var_795, y = input_123)[name = tensor("input_133")]; tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; @@ -752,19 +759,19 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_806 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; - tensor var_807 = const()[name = tensor("op_807"), val = tensor([1, 1, 4, 64])]; - tensor var_808 = reshape(shape = var_807, x = var_806)[name = tensor("op_808")]; + tensor var_809 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_810 = const()[name = tensor("op_810"), val = tensor([1, 1, 4, 64])]; + tensor var_811 = reshape(shape = var_810, x = var_809)[name = tensor("op_811")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_812 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; - tensor var_813 = const()[name = tensor("op_813"), val = tensor(0x1p-3)]; - tensor var_814 = mul(x = var_812, y = var_813)[name = tensor("op_814")]; - tensor var_815 = const()[name = tensor("op_815"), val = tensor([1, 1, 4, 64])]; - tensor var_816 = reshape(shape = var_815, x = var_814)[name = tensor("op_816")]; + tensor var_815 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_816 = const()[name = tensor("op_816"), val = tensor(0x1p-3)]; + tensor var_817 = mul(x = var_815, y = var_816)[name = tensor("op_817")]; + tensor var_818 = const()[name = tensor("op_818"), val = tensor([1, 1, 4, 64])]; + tensor var_819 = reshape(shape = var_818, x = var_817)[name = tensor("op_819")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_820 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; - tensor var_821 = const()[name = tensor("op_821"), val = tensor([1, 1, 4, 64])]; - tensor var_822 = reshape(shape = var_821, x = var_820)[name = tensor("op_822")]; + tensor var_823 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_824 = const()[name = tensor("op_824"), val = tensor([1, 1, 4, 64])]; + tensor var_825 = reshape(shape = var_824, x = var_823)[name = tensor("op_825")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; @@ -772,84 +779,86 @@ program(1.0) tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_816)[name = tensor("transpose_35")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_808)[name = tensor("transpose_36")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_819)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_811)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_832 = const()[name = tensor("op_832"), val = tensor([1, 1])]; - tensor var_833 = reshape(shape = var_832, x = sqrt_s_t_7)[name = tensor("op_833")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_833)[name = tensor("M_7")]; - tensor var_835 = mul(x = qk_7, y = M_7)[name = tensor("op_835")]; + tensor var_835 = const()[name = tensor("op_835"), val = tensor([1, 1])]; + tensor var_836 = reshape(shape = var_835, x = sqrt_s_t_7)[name = tensor("op_836")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_836)[name = tensor("M_7")]; + tensor var_838 = mul(x = qk_7, y = M_7)[name = tensor("op_838")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_822)[name = tensor("transpose_34")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_835, y = v_7)[name = tensor("inner_7")]; - tensor var_837_transpose_x_0 = const()[name = tensor("op_837_transpose_x_0"), val = tensor(false)]; - tensor var_837_transpose_y_0 = const()[name = tensor("op_837_transpose_y_0"), val = tensor(false)]; - tensor var_837 = matmul(transpose_x = var_837_transpose_x_0, transpose_y = var_837_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_837")]; - tensor var_838 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_838")]; - tensor var_839 = const()[name = tensor("op_839"), val = tensor([1, 1, 1, 1])]; - tensor var_840 = reshape(shape = var_839, x = var_838)[name = tensor("op_840")]; - tensor cross_7 = mul(x = var_837, y = var_840)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_825)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_838, y = v_7)[name = tensor("inner_7")]; + tensor var_840_transpose_x_0 = const()[name = tensor("op_840_transpose_x_0"), val = tensor(false)]; + tensor var_840_transpose_y_0 = const()[name = tensor("op_840_transpose_y_0"), val = tensor(false)]; + tensor var_840 = matmul(transpose_x = var_840_transpose_x_0, transpose_y = var_840_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_840")]; + tensor var_841 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_841")]; + tensor var_842 = const()[name = tensor("op_842"), val = tensor([1, 1, 1, 1])]; + tensor var_843 = reshape(shape = var_842, x = var_841)[name = tensor("op_843")]; + tensor cross_7 = mul(x = var_840, y = var_843)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_843 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_843")]; - tensor var_845_transpose_x_1 = const()[name = tensor("op_845_transpose_x_1"), val = tensor(true)]; - tensor var_845_transpose_y_1 = const()[name = tensor("op_845_transpose_y_1"), val = tensor(false)]; - tensor var_845 = matmul(transpose_x = var_845_transpose_x_1, transpose_y = var_845_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_845")]; - tensor new_kv_unnorm_7 = add(x = var_843, y = var_845)[name = tensor("new_kv_unnorm_7")]; - tensor var_847 = const()[name = tensor("op_847"), val = tensor(0x1p+0)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_847)[name = tensor("new_scale_7")]; - tensor var_849 = sqrt(x = new_scale_7)[name = tensor("op_849")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_849)[name = tensor("nkv_1")]; - tensor var_851_perm_0 = const()[name = tensor("op_851_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_846 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_846")]; + tensor var_848_transpose_x_1 = const()[name = tensor("op_848_transpose_x_1"), val = tensor(true)]; + tensor var_848_transpose_y_1 = const()[name = tensor("op_848_transpose_y_1"), val = tensor(false)]; + tensor var_848 = matmul(transpose_x = var_848_transpose_x_1, transpose_y = var_848_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_848")]; + tensor new_kv_unnorm_7 = add(x = var_846, y = var_848)[name = tensor("new_kv_unnorm_7")]; + tensor var_850 = const()[name = tensor("op_850"), val = tensor(0x1p+0)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_850)[name = tensor("new_scale_7")]; + tensor var_852 = sqrt(x = new_scale_7)[name = tensor("op_852")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_852)[name = tensor("nkv_1")]; + tensor var_854_perm_0 = const()[name = tensor("op_854_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_851 = transpose(perm = var_851_perm_0, x = out_19)[name = tensor("transpose_33")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_851)[name = tensor("out_21")]; - tensor var_855 = const()[name = tensor("op_855"), val = tensor([1, 1, 256])]; - tensor out_23 = reshape(shape = var_855, x = out_21)[name = tensor("out_23")]; - tensor var_857 = silu(x = input_137)[name = tensor("op_857")]; - tensor input_139 = mul(x = var_857, y = out_23)[name = tensor("input_139")]; + tensor var_854 = transpose(perm = var_854_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_854)[name = tensor("out_21")]; + tensor var_858 = const()[name = tensor("op_858"), val = tensor([1, 1, 256])]; + tensor out_23 = reshape(shape = var_858, x = out_21)[name = tensor("out_23")]; + tensor var_860 = silu(x = input_137)[name = tensor("op_860")]; + tensor input_139 = mul(x = var_860, y = out_23)[name = tensor("input_139")]; tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor window_13_begin_0 = const()[name = tensor("window_13_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_13_end_0 = const()[name = tensor("window_13_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_13_end_mask_0 = const()[name = tensor("window_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_13_squeeze_mask_0 = const()[name = tensor("window_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_13 = slice_by_index(begin = window_13_begin_0, end = window_13_end_0, end_mask = window_13_end_mask_0, squeeze_mask = window_13_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_13")]; tensor filled_7_begin_0 = const()[name = tensor("filled_7_begin_0"), val = tensor([3, 0])]; tensor filled_7_end_0 = const()[name = tensor("filled_7_end_0"), val = tensor([4, 1])]; tensor filled_7_end_mask_0 = const()[name = tensor("filled_7_end_mask_0"), val = tensor([false, true])]; tensor filled_7_squeeze_mask_0 = const()[name = tensor("filled_7_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_7 = slice_by_index(begin = filled_7_begin_0, end = filled_7_end_0, end_mask = filled_7_end_mask_0, squeeze_mask = filled_7_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_7")]; - tensor var_869_begin_0 = const()[name = tensor("op_869_begin_0"), val = tensor([0, 1, 0])]; - tensor var_869_end_0 = const()[name = tensor("op_869_end_0"), val = tensor([1, 16, 256])]; - tensor var_869_end_mask_0 = const()[name = tensor("op_869_end_mask_0"), val = tensor([true, true, true])]; - tensor var_869 = slice_by_index(begin = var_869_begin_0, end = var_869_end_0, end_mask = var_869_end_mask_0, x = window)[name = tensor("op_869")]; + tensor var_872_begin_0 = const()[name = tensor("op_872_begin_0"), val = tensor([0, 1, 0])]; + tensor var_872_end_0 = const()[name = tensor("op_872_end_0"), val = tensor([1, 16, 256])]; + tensor var_872_end_mask_0 = const()[name = tensor("op_872_end_mask_0"), val = tensor([true, true, true])]; + tensor var_872 = slice_by_index(begin = var_872_begin_0, end = var_872_end_0, end_mask = var_872_end_mask_0, x = window_13)[name = tensor("op_872")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_869, x_21))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_872, x_21))[name = tensor("rolled")]; tensor clip_6 = clip(alpha = var_21, beta = var_20, x = filled_7)[name = tensor("clip_6")]; - tensor var_874 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_874")]; - tensor var_875_dtype_0 = const()[name = tensor("op_875_dtype_0"), val = tensor("fp32")]; - tensor var_876 = const()[name = tensor("op_876"), val = tensor([1, 16, 1])]; - tensor var_875 = cast(dtype = var_875_dtype_0, x = var_874)[name = tensor("cast_121")]; - tensor fill_mask = reshape(shape = var_876, x = var_875)[name = tensor("fill_mask")]; + tensor var_877 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_877")]; + tensor var_878_dtype_0 = const()[name = tensor("op_878_dtype_0"), val = tensor("fp32")]; + tensor var_879 = const()[name = tensor("op_879"), val = tensor([1, 16, 1])]; + tensor var_878 = cast(dtype = var_878_dtype_0, x = var_877)[name = tensor("cast_121")]; + tensor fill_mask = reshape(shape = var_879, x = var_878)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; tensor x_expanded = tile(reps = x_expanded_reps_0, x = x_21)[name = tensor("x_expanded")]; - tensor var_880 = sub(x = var_17, y = fill_mask)[name = tensor("op_880")]; - tensor var_881 = mul(x = window, y = var_880)[name = tensor("op_881")]; - tensor var_882 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_882")]; - tensor filled_window = add(x = var_881, y = var_882)[name = tensor("filled_window")]; - tensor var_884 = greater_equal(x = filled_7, y = var_16)[name = tensor("op_884")]; - tensor var_885_dtype_0 = const()[name = tensor("op_885_dtype_0"), val = tensor("fp32")]; - tensor var_886 = const()[name = tensor("op_886"), val = tensor([1, 1, 1])]; - tensor var_885 = cast(dtype = var_885_dtype_0, x = var_884)[name = tensor("cast_120")]; - tensor use_roll = reshape(shape = var_886, x = var_885)[name = tensor("use_roll")]; - tensor var_888 = sub(x = var_17, y = use_roll)[name = tensor("op_888")]; - tensor var_889 = mul(x = filled_window, y = var_888)[name = tensor("op_889")]; - tensor var_890 = mul(x = rolled, y = use_roll)[name = tensor("op_890")]; - tensor input_141 = add(x = var_889, y = var_890)[name = tensor("input_141")]; - tensor var_892 = const()[name = tensor("op_892"), val = tensor(0x1p+0)]; - tensor var_893 = add(x = filled_7, y = var_892)[name = tensor("op_893")]; - tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_893)[name = tensor("clip_7")]; + tensor var_883 = sub(x = var_17, y = fill_mask)[name = tensor("op_883")]; + tensor var_884 = mul(x = window_13, y = var_883)[name = tensor("op_884")]; + tensor var_885 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_885")]; + tensor filled_window = add(x = var_884, y = var_885)[name = tensor("filled_window")]; + tensor var_887 = greater_equal(x = filled_7, y = var_16)[name = tensor("op_887")]; + tensor var_888_dtype_0 = const()[name = tensor("op_888_dtype_0"), val = tensor("fp32")]; + tensor var_889 = const()[name = tensor("op_889"), val = tensor([1, 1, 1])]; + tensor var_888 = cast(dtype = var_888_dtype_0, x = var_887)[name = tensor("cast_120")]; + tensor use_roll = reshape(shape = var_889, x = var_888)[name = tensor("use_roll")]; + tensor var_891 = sub(x = var_17, y = use_roll)[name = tensor("op_891")]; + tensor var_892 = mul(x = filled_window, y = var_891)[name = tensor("op_892")]; + tensor var_893 = mul(x = rolled, y = use_roll)[name = tensor("op_893")]; + tensor window = add(x = var_892, y = var_893)[name = tensor("window")]; + tensor var_895 = const()[name = tensor("op_895"), val = tensor(0x1p+0)]; + tensor var_896 = add(x = filled_7, y = var_895)[name = tensor("op_896")]; + tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_896)[name = tensor("clip_7")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_30, interleave = input_141_interleave_0, values = window)[name = tensor("input_141")]; tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; @@ -858,13 +867,13 @@ program(1.0) tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; - tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_32")]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; - tensor var_914_split_sizes_0 = const()[name = tensor("op_914_split_sizes_0"), val = tensor([256, 256])]; - tensor var_914_axis_0 = const()[name = tensor("op_914_axis_0"), val = tensor(1)]; - tensor var_914_0, tensor var_914_1 = split(axis = var_914_axis_0, split_sizes = var_914_split_sizes_0, x = inputs_33)[name = tensor("op_914")]; - tensor var_916 = sigmoid(x = var_914_1)[name = tensor("op_916")]; - tensor inputs_35 = mul(x = var_914_0, y = var_916)[name = tensor("inputs_35")]; + tensor var_919_split_sizes_0 = const()[name = tensor("op_919_split_sizes_0"), val = tensor([256, 256])]; + tensor var_919_axis_0 = const()[name = tensor("op_919_axis_0"), val = tensor(1)]; + tensor var_919_0, tensor var_919_1 = split(axis = var_919_axis_0, split_sizes = var_919_split_sizes_0, x = inputs_33)[name = tensor("op_919")]; + tensor var_921 = sigmoid(x = var_919_1)[name = tensor("op_921")]; + tensor inputs_35 = mul(x = var_919_0, y = var_921)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; @@ -884,22 +893,22 @@ program(1.0) tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_947_begin_0 = const()[name = tensor("op_947_begin_0"), val = tensor([0, -1, 0])]; - tensor var_947_end_0 = const()[name = tensor("op_947_end_0"), val = tensor([1, 16, 256])]; - tensor var_947_end_mask_0 = const()[name = tensor("op_947_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_31")]; - tensor var_947 = slice_by_index(begin = var_947_begin_0, end = var_947_end_0, end_mask = var_947_end_mask_0, x = conv_out_7)[name = tensor("op_947")]; - tensor var_950_interleave_0 = const()[name = tensor("op_950_interleave_0"), val = tensor(false)]; - tensor var_950 = concat(axis = var_33, interleave = var_950_interleave_0, values = var_947)[name = tensor("op_950")]; - tensor input_151 = add(x = x_21, y = var_950)[name = tensor("input_151")]; + tensor var_952_begin_0 = const()[name = tensor("op_952_begin_0"), val = tensor([0, -1, 0])]; + tensor var_952_end_0 = const()[name = tensor("op_952_end_0"), val = tensor([1, 16, 256])]; + tensor var_952_end_mask_0 = const()[name = tensor("op_952_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_952 = slice_by_index(begin = var_952_begin_0, end = var_952_end_0, end_mask = var_952_end_mask_0, x = conv_out_7)[name = tensor("op_952")]; + tensor var_954_perm_0 = const()[name = tensor("op_954_perm_0"), val = tensor([1, 0, 2])]; + tensor var_954 = transpose(perm = var_954_perm_0, x = var_952)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_954)[name = tensor("input_151")]; tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; - tensor var_973 = const()[name = tensor("op_973"), val = tensor(0x1p-1)]; - tensor var_974 = mul(x = input_159, y = var_973)[name = tensor("op_974")]; - tensor input_161 = add(x = var_974, y = input_151)[name = tensor("input_161")]; + tensor var_977 = const()[name = tensor("op_977"), val = tensor(0x1p-1)]; + tensor var_978 = mul(x = input_159, y = var_977)[name = tensor("op_978")]; + tensor input_161 = add(x = var_978, y = input_151)[name = tensor("input_161")]; tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; @@ -912,44 +921,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_992_begin_0 = const()[name = tensor("op_992_begin_0"), val = tensor([0, 0, 1])]; - tensor var_992_end_0 = const()[name = tensor("op_992_end_0"), val = tensor([1, 256, 19])]; - tensor var_992_end_mask_0 = const()[name = tensor("op_992_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_992_begin_0, end = var_992_end_0, end_mask = var_992_end_mask_0, x = cat)[name = tensor("op_992")]; + tensor var_996_begin_0 = const()[name = tensor("op_996_begin_0"), val = tensor([0, 0, 1])]; + tensor var_996_end_0 = const()[name = tensor("op_996_end_0"), val = tensor([1, 256, 19])]; + tensor var_996_end_mask_0 = const()[name = tensor("op_996_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_996_begin_0, end = var_996_end_0, end_mask = var_996_end_mask_0, x = cat)[name = tensor("op_996")]; tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; - tensor var_994 = const()[name = tensor("op_994"), val = tensor([-1])]; + tensor var_998 = const()[name = tensor("op_998"), val = tensor([-1])]; tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_995 = reduce_l2_norm(axes = var_994, keep_dims = var_36, x = input_163)[name = tensor("op_995")]; + tensor var_999 = reduce_l2_norm(axes = var_998, keep_dims = var_36, x = input_163)[name = tensor("op_999")]; tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; - tensor clip_8 = clip(alpha = var_12, beta = const_12, x = var_995)[name = tensor("clip_8")]; + tensor clip_8 = clip(alpha = var_12, beta = const_12, x = var_999)[name = tensor("clip_8")]; tensor emb = real_div(x = input_163, y = clip_8)[name = tensor("emb")]; - tensor var_999_axis_0 = const()[name = tensor("op_999_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_999_axis_0, values = (var_214, var_426, var_638, nkv_1))[name = tensor("op_999")]; - tensor var_1001_axis_0 = const()[name = tensor("op_1001_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_1001_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1001")]; tensor var_1003_axis_0 = const()[name = tensor("op_1003_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_1003_axis_0, values = (input_21, input_61, input_101, input_141))[name = tensor("op_1003")]; + tensor enc_kv_new = stack(axis = var_1003_axis_0, values = (var_214, var_427, var_640, nkv_1))[name = tensor("op_1003")]; tensor var_1005_axis_0 = const()[name = tensor("op_1005_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_1005_axis_0, values = (clip_1, clip_3, clip_5, clip_7))[name = tensor("op_1005")]; - tensor var_1015 = const()[name = tensor("op_1015"), val = tensor(0x1.5798eep-27)]; - tensor var_1021 = const()[name = tensor("op_1021"), val = tensor(0x1.4f8b58p-17)]; - tensor var_1023 = const()[name = tensor("op_1023"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_1024 = const()[name = tensor("op_1024"), val = tensor(true)]; - tensor var_1026 = const()[name = tensor("op_1026"), val = tensor(0x1p+0)]; - tensor var_1030 = const()[name = tensor("op_1030"), val = tensor(-1)]; - tensor var_1035 = const()[name = tensor("op_1035"), val = tensor(0)]; - tensor var_1092 = const()[name = tensor("op_1092"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395456)))]; - tensor var_1097_axes_0 = const()[name = tensor("op_1097_axes_0"), val = tensor([2])]; - tensor var_1097 = expand_dims(axes = var_1097_axes_0, x = emb)[name = tensor("op_1097")]; - tensor var_1098 = const()[name = tensor("op_1098"), val = tensor([1, 1, 6, 1])]; - tensor emb_exp = tile(reps = var_1098, x = var_1097)[name = tensor("emb_exp")]; + tensor enc_scale_new = stack(axis = var_1005_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1005")]; + tensor var_1007_axis_0 = const()[name = tensor("op_1007_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1007_axis_0, values = (window_3, window_7, window_11, window))[name = tensor("op_1007")]; + tensor var_1009_axis_0 = const()[name = tensor("op_1009_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1009_axis_0, values = (clip_1, clip_3, clip_5, clip_7))[name = tensor("op_1009")]; + tensor var_1019 = const()[name = tensor("op_1019"), val = tensor(0x1.5798eep-27)]; + tensor var_1024 = const()[name = tensor("op_1024"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1026 = const()[name = tensor("op_1026"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1027 = const()[name = tensor("op_1027"), val = tensor(true)]; + tensor var_1029 = const()[name = tensor("op_1029"), val = tensor(0x1p+0)]; + tensor var_1033 = const()[name = tensor("op_1033"), val = tensor(-1)]; + tensor var_1039 = const()[name = tensor("op_1039"), val = tensor(0)]; + tensor var_1096 = const()[name = tensor("op_1096"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395456)))]; + tensor var_1101_axes_0 = const()[name = tensor("op_1101_axes_0"), val = tensor([2])]; + tensor var_1101 = expand_dims(axes = var_1101_axes_0, x = emb)[name = tensor("op_1101")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 6, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1101)[name = tensor("emb_exp")]; tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; - tensor input_165 = concat(axis = var_1030, interleave = input_165_interleave_0, values = (emb_exp, var_1092))[name = tensor("input_165")]; + tensor input_165 = concat(axis = var_1033, interleave = input_165_interleave_0, values = (emb_exp, var_1096))[name = tensor("input_165")]; tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; - tensor var_1105_perm_0 = const()[name = tensor("op_1105_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1109 = const()[name = tensor("op_1109"), val = tensor([6, 1, 256])]; - tensor var_1105 = transpose(perm = var_1105_perm_0, x = x_27)[name = tensor("transpose_28")]; - tensor x_29 = reshape(shape = var_1109, x = var_1105)[name = tensor("x_29")]; + tensor var_1109_perm_0 = const()[name = tensor("op_1109_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1113 = const()[name = tensor("op_1113"), val = tensor([6, 1, 256])]; + tensor var_1109 = transpose(perm = var_1109_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1113, x = var_1109)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 6, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -960,131 +969,131 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_1117 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; - tensor var_1118 = const()[name = tensor("op_1118"), val = tensor([6, 1, 4, 64])]; - tensor var_1119 = reshape(shape = var_1118, x = var_1117)[name = tensor("op_1119")]; + tensor var_1121 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1122 = const()[name = tensor("op_1122"), val = tensor([6, 1, 4, 64])]; + tensor var_1123 = reshape(shape = var_1122, x = var_1121)[name = tensor("op_1123")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1123 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; - tensor var_1124 = const()[name = tensor("op_1124"), val = tensor(0x1p-3)]; - tensor var_1125 = mul(x = var_1123, y = var_1124)[name = tensor("op_1125")]; - tensor var_1126 = const()[name = tensor("op_1126"), val = tensor([6, 1, 4, 64])]; - tensor var_1127 = reshape(shape = var_1126, x = var_1125)[name = tensor("op_1127")]; + tensor var_1127 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1128 = const()[name = tensor("op_1128"), val = tensor(0x1p-3)]; + tensor var_1129 = mul(x = var_1127, y = var_1128)[name = tensor("op_1129")]; + tensor var_1130 = const()[name = tensor("op_1130"), val = tensor([6, 1, 4, 64])]; + tensor var_1131 = reshape(shape = var_1130, x = var_1129)[name = tensor("op_1131")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1131 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; - tensor var_1132 = const()[name = tensor("op_1132"), val = tensor([6, 1, 4, 64])]; - tensor var_1133 = reshape(shape = var_1132, x = var_1131)[name = tensor("op_1133")]; + tensor var_1135 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1136 = const()[name = tensor("op_1136"), val = tensor([6, 1, 4, 64])]; + tensor var_1137 = reshape(shape = var_1136, x = var_1135)[name = tensor("op_1137")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_1035, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1039, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; - tensor clip_9 = clip(alpha = var_1026, beta = const_20, x = s_t_9)[name = tensor("clip_9")]; + tensor clip_9 = clip(alpha = var_1029, beta = const_20, x = s_t_9)[name = tensor("clip_9")]; tensor sqrt_s_t_9 = sqrt(x = clip_9)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_1127)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_1119)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1131)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1123)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_1145 = const()[name = tensor("op_1145"), val = tensor([1, 1])]; - tensor var_1146 = reshape(shape = var_1145, x = valid_mask)[name = tensor("op_1146")]; - tensor var_1148 = const()[name = tensor("op_1148"), val = tensor([1, 1])]; - tensor var_1149 = reshape(shape = var_1148, x = sqrt_s_t_9)[name = tensor("op_1149")]; - tensor M_9 = real_div(x = var_1146, y = var_1149)[name = tensor("M_9")]; - tensor var_1151 = mul(x = qk_9, y = M_9)[name = tensor("op_1151")]; + tensor var_1149 = const()[name = tensor("op_1149"), val = tensor([1, 1])]; + tensor var_1150 = reshape(shape = var_1149, x = valid_mask)[name = tensor("op_1150")]; + tensor var_1152 = const()[name = tensor("op_1152"), val = tensor([1, 1])]; + tensor var_1153 = reshape(shape = var_1152, x = sqrt_s_t_9)[name = tensor("op_1153")]; + tensor M_9 = real_div(x = var_1150, y = var_1153)[name = tensor("M_9")]; + tensor var_1155 = mul(x = qk_9, y = M_9)[name = tensor("op_1155")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_1133)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1151, y = v_9)[name = tensor("inner_9")]; - tensor var_1153_transpose_x_0 = const()[name = tensor("op_1153_transpose_x_0"), val = tensor(false)]; - tensor var_1153_transpose_y_0 = const()[name = tensor("op_1153_transpose_y_0"), val = tensor(false)]; - tensor var_1153 = matmul(transpose_x = var_1153_transpose_x_0, transpose_y = var_1153_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1153")]; - tensor var_1154 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1154")]; - tensor var_1155 = const()[name = tensor("op_1155"), val = tensor([1, 1, 1, 1])]; - tensor var_1156 = reshape(shape = var_1155, x = var_1154)[name = tensor("op_1156")]; - tensor cross_9 = mul(x = var_1153, y = var_1156)[name = tensor("cross_9")]; - tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1137)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1155, y = v_9)[name = tensor("inner_9")]; + tensor var_1157_transpose_x_0 = const()[name = tensor("op_1157_transpose_x_0"), val = tensor(false)]; + tensor var_1157_transpose_y_0 = const()[name = tensor("op_1157_transpose_y_0"), val = tensor(false)]; + tensor var_1157 = matmul(transpose_x = var_1157_transpose_x_0, transpose_y = var_1157_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1157")]; + tensor var_1158 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1158")]; tensor var_1159 = const()[name = tensor("op_1159"), val = tensor([1, 1, 1, 1])]; - tensor var_1160 = reshape(shape = var_1159, x = valid_mask)[name = tensor("op_1160")]; - tensor v_masked_1 = mul(x = v_9, y = var_1160)[name = tensor("v_masked_1")]; - tensor var_1162 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1162")]; - tensor var_1164_transpose_x_1 = const()[name = tensor("op_1164_transpose_x_1"), val = tensor(true)]; - tensor var_1164_transpose_y_1 = const()[name = tensor("op_1164_transpose_y_1"), val = tensor(false)]; - tensor var_1164 = matmul(transpose_x = var_1164_transpose_x_1, transpose_y = var_1164_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1164")]; - tensor new_kv_unnorm_9 = add(x = var_1162, y = var_1164)[name = tensor("new_kv_unnorm_9")]; - tensor var_1166_keep_dims_0 = const()[name = tensor("op_1166_keep_dims_0"), val = tensor(false)]; - tensor var_1166 = reduce_sum(keep_dims = var_1166_keep_dims_0, x = valid_mask)[name = tensor("op_1166")]; - tensor var_1167 = const()[name = tensor("op_1167"), val = tensor([1])]; - tensor var_1168 = reshape(shape = var_1167, x = var_1166)[name = tensor("op_1168")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_1168)[name = tensor("new_scale_9")]; + tensor var_1160 = reshape(shape = var_1159, x = var_1158)[name = tensor("op_1160")]; + tensor cross_9 = mul(x = var_1157, y = var_1160)[name = tensor("cross_9")]; + tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; + tensor var_1163 = const()[name = tensor("op_1163"), val = tensor([1, 1, 1, 1])]; + tensor var_1164 = reshape(shape = var_1163, x = valid_mask)[name = tensor("op_1164")]; + tensor v_masked_1 = mul(x = v_9, y = var_1164)[name = tensor("v_masked_1")]; + tensor var_1166 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1166")]; + tensor var_1168_transpose_x_1 = const()[name = tensor("op_1168_transpose_x_1"), val = tensor(true)]; + tensor var_1168_transpose_y_1 = const()[name = tensor("op_1168_transpose_y_1"), val = tensor(false)]; + tensor var_1168 = matmul(transpose_x = var_1168_transpose_x_1, transpose_y = var_1168_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1168")]; + tensor new_kv_unnorm_9 = add(x = var_1166, y = var_1168)[name = tensor("new_kv_unnorm_9")]; + tensor var_1170_keep_dims_0 = const()[name = tensor("op_1170_keep_dims_0"), val = tensor(false)]; + tensor var_1170 = reduce_sum(keep_dims = var_1170_keep_dims_0, x = valid_mask)[name = tensor("op_1170")]; + tensor var_1171 = const()[name = tensor("op_1171"), val = tensor([1])]; + tensor var_1172 = reshape(shape = var_1171, x = var_1170)[name = tensor("op_1172")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1172)[name = tensor("new_scale_9")]; tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; - tensor clip_10 = clip(alpha = var_1026, beta = const_21, x = new_scale_9)[name = tensor("clip_10")]; + tensor clip_10 = clip(alpha = var_1029, beta = const_21, x = new_scale_9)[name = tensor("clip_10")]; tensor sqrt_new_scale_1 = sqrt(x = clip_10)[name = tensor("sqrt_new_scale_1")]; - tensor var_1172 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1172")]; - tensor var_1173_perm_0 = const()[name = tensor("op_1173_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1176 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1176")]; + tensor var_1177_perm_0 = const()[name = tensor("op_1177_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_1173 = transpose(perm = var_1173_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1023, x = var_1173)[name = tensor("out_27")]; - tensor var_1177 = const()[name = tensor("op_1177"), val = tensor([6, 1, 256])]; - tensor out_29 = reshape(shape = var_1177, x = out_27)[name = tensor("out_29")]; - tensor var_1179 = silu(x = input_169)[name = tensor("op_1179")]; - tensor input_171 = mul(x = var_1179, y = out_29)[name = tensor("input_171")]; + tensor var_1177 = transpose(perm = var_1177_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1026, x = var_1177)[name = tensor("out_27")]; + tensor var_1181 = const()[name = tensor("op_1181"), val = tensor([6, 1, 256])]; + tensor out_29 = reshape(shape = var_1181, x = out_27)[name = tensor("out_29")]; + tensor var_1183 = silu(x = input_169)[name = tensor("op_1183")]; + tensor input_171 = mul(x = var_1183, y = out_29)[name = tensor("input_171")]; tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1021, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; - tensor var_1189 = const()[name = tensor("op_1189"), val = tensor([1, 6, 1, 256])]; - tensor var_1190 = reshape(shape = var_1189, x = xt_1)[name = tensor("op_1190")]; - tensor var_1191_perm_0 = const()[name = tensor("op_1191_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1194 = const()[name = tensor("op_1194"), val = tensor([1, 6, 256])]; - tensor var_1191 = transpose(perm = var_1191_perm_0, x = var_1190)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_1194, x = var_1191)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1024, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1193 = const()[name = tensor("op_1193"), val = tensor([1, 6, 1, 256])]; + tensor var_1194 = reshape(shape = var_1193, x = xt_1)[name = tensor("op_1194")]; + tensor var_1195_perm_0 = const()[name = tensor("op_1195_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1198 = const()[name = tensor("op_1198"), val = tensor([1, 6, 256])]; + tensor var_1195 = transpose(perm = var_1195_perm_0, x = var_1194)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1198, x = var_1195)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_1217 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_4 = const()[name = tensor("concat_4"), val = tensor([6, 1, 3, 256])]; - tensor var_1219 = reshape(shape = concat_4, x = var_1217)[name = tensor("op_1219")]; - tensor var_1220_axes_0 = const()[name = tensor("op_1220_axes_0"), val = tensor([0])]; - tensor var_1220 = expand_dims(axes = var_1220_axes_0, x = var_1219)[name = tensor("op_1220")]; - tensor var_1221_perm_0 = const()[name = tensor("op_1221_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1222_axes_0 = const()[name = tensor("op_1222_axes_0"), val = tensor([-2])]; - tensor var_1221 = transpose(perm = var_1221_perm_0, x = var_1220)[name = tensor("transpose_21")]; - tensor var_1222 = squeeze(axes = var_1222_axes_0, x = var_1221)[name = tensor("op_1222")]; + tensor var_1221 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_5 = const()[name = tensor("concat_5"), val = tensor([6, 1, 3, 256])]; + tensor var_1223 = reshape(shape = concat_5, x = var_1221)[name = tensor("op_1223")]; + tensor var_1224_axes_0 = const()[name = tensor("op_1224_axes_0"), val = tensor([0])]; + tensor var_1224 = expand_dims(axes = var_1224_axes_0, x = var_1223)[name = tensor("op_1224")]; + tensor var_1225_perm_0 = const()[name = tensor("op_1225_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1226_axes_0 = const()[name = tensor("op_1226_axes_0"), val = tensor([-2])]; + tensor var_1225 = transpose(perm = var_1225_perm_0, x = var_1224)[name = tensor("transpose_21")]; + tensor var_1226 = squeeze(axes = var_1226_axes_0, x = var_1225)[name = tensor("op_1226")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 6, 1, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1222)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1226)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 6, 1, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1222)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1226)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 6, 1, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1222)[name = tensor("v_11")]; - tensor var_1230 = const()[name = tensor("op_1230"), val = tensor([6, 4, 64])]; - tensor var_1231 = reshape(shape = var_1230, x = q_11)[name = tensor("op_1231")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1226)[name = tensor("v_11")]; + tensor var_1234 = const()[name = tensor("op_1234"), val = tensor([6, 4, 64])]; + tensor var_1235 = reshape(shape = var_1234, x = q_11)[name = tensor("op_1235")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1237 = const()[name = tensor("op_1237"), val = tensor([6, 4, 64])]; - tensor var_1238 = reshape(shape = var_1237, x = k_11)[name = tensor("op_1238")]; + tensor var_1241 = const()[name = tensor("op_1241"), val = tensor([6, 4, 64])]; + tensor var_1242 = reshape(shape = var_1241, x = k_11)[name = tensor("op_1242")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1244 = const()[name = tensor("op_1244"), val = tensor([6, 4, 64])]; - tensor var_1245 = reshape(shape = var_1244, x = v_11)[name = tensor("op_1245")]; + tensor var_1248 = const()[name = tensor("op_1248"), val = tensor([6, 4, 64])]; + tensor var_1249 = reshape(shape = var_1248, x = v_11)[name = tensor("op_1249")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1248 = const()[name = tensor("op_1248"), val = tensor([1, 4, 6, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_1231)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_1248, x = q_13)[name = tensor("q_15")]; - tensor var_1250 = const()[name = tensor("op_1250"), val = tensor([1, 4, 6, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_1238)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_1250, x = k_13)[name = tensor("k_15")]; tensor var_1252 = const()[name = tensor("op_1252"), val = tensor([1, 4, 6, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_1245)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_1252, x = v_13)[name = tensor("v_15")]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1235)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1252, x = q_13)[name = tensor("q_15")]; + tensor var_1254 = const()[name = tensor("op_1254"), val = tensor([1, 4, 6, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1242)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1254, x = k_13)[name = tensor("k_15")]; + tensor var_1256 = const()[name = tensor("op_1256"), val = tensor([1, 4, 6, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1249)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1256, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -1095,30 +1104,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_1255 = const()[name = tensor("op_1255"), val = tensor([2, 0, 1, 3])]; - tensor var_1260 = const()[name = tensor("op_1260"), val = tensor([6, 256])]; - tensor var_1256 = transpose(perm = var_1255, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_1260, x = var_1256)[name = tensor("attn_output_3")]; + tensor var_1259 = const()[name = tensor("op_1259"), val = tensor([2, 0, 1, 3])]; + tensor var_1264 = const()[name = tensor("op_1264"), val = tensor([6, 256])]; + tensor var_1260 = transpose(perm = var_1259, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1264, x = var_1260)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_1264 = const()[name = tensor("op_1264"), val = tensor([6, 1, 256])]; - tensor attn_output_7 = reshape(shape = var_1264, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1268 = const()[name = tensor("op_1268"), val = tensor([6, 1, 256])]; + tensor attn_output_7 = reshape(shape = var_1268, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; - tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1021, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1024, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1021, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; - tensor var_1284 = const()[name = tensor("op_1284"), val = tensor([1, 1, 6, 256])]; - tensor x_31 = reshape(shape = var_1284, x = xt_3)[name = tensor("x_31")]; - tensor var_1286_perm_0 = const()[name = tensor("op_1286_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1290 = const()[name = tensor("op_1290"), val = tensor([6, 1, 256])]; - tensor var_1286 = transpose(perm = var_1286_perm_0, x = x_31)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_1290, x = var_1286)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1024, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1288 = const()[name = tensor("op_1288"), val = tensor([1, 1, 6, 256])]; + tensor x_31 = reshape(shape = var_1288, x = xt_3)[name = tensor("x_31")]; + tensor var_1290_perm_0 = const()[name = tensor("op_1290_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1294 = const()[name = tensor("op_1294"), val = tensor([6, 1, 256])]; + tensor var_1290 = transpose(perm = var_1290_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1294, x = var_1290)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 6, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1129,120 +1138,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_1298 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_1299 = const()[name = tensor("op_1299"), val = tensor([6, 1, 4, 64])]; - tensor var_1300 = reshape(shape = var_1299, x = var_1298)[name = tensor("op_1300")]; + tensor var_1302 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1303 = const()[name = tensor("op_1303"), val = tensor([6, 1, 4, 64])]; + tensor var_1304 = reshape(shape = var_1303, x = var_1302)[name = tensor("op_1304")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1304 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_1305 = const()[name = tensor("op_1305"), val = tensor(0x1p-3)]; - tensor var_1306 = mul(x = var_1304, y = var_1305)[name = tensor("op_1306")]; - tensor var_1307 = const()[name = tensor("op_1307"), val = tensor([6, 1, 4, 64])]; - tensor var_1308 = reshape(shape = var_1307, x = var_1306)[name = tensor("op_1308")]; + tensor var_1308 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1309 = const()[name = tensor("op_1309"), val = tensor(0x1p-3)]; + tensor var_1310 = mul(x = var_1308, y = var_1309)[name = tensor("op_1310")]; + tensor var_1311 = const()[name = tensor("op_1311"), val = tensor([6, 1, 4, 64])]; + tensor var_1312 = reshape(shape = var_1311, x = var_1310)[name = tensor("op_1312")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1312 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_1313 = const()[name = tensor("op_1313"), val = tensor([6, 1, 4, 64])]; - tensor var_1314 = reshape(shape = var_1313, x = var_1312)[name = tensor("op_1314")]; + tensor var_1316 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1317 = const()[name = tensor("op_1317"), val = tensor([6, 1, 4, 64])]; + tensor var_1318 = reshape(shape = var_1317, x = var_1316)[name = tensor("op_1318")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; - tensor clip_11 = clip(alpha = var_1026, beta = const_32, x = s_t)[name = tensor("clip_11")]; + tensor clip_11 = clip(alpha = var_1029, beta = const_32, x = s_t)[name = tensor("clip_11")]; tensor sqrt_s_t = sqrt(x = clip_11)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_1308)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_1300)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1312)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1304)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_1329 = const()[name = tensor("op_1329"), val = tensor([1, 1])]; - tensor var_1330 = reshape(shape = var_1329, x = sqrt_s_t)[name = tensor("op_1330")]; - tensor M = real_div(x = var_1146, y = var_1330)[name = tensor("M")]; - tensor var_1332 = mul(x = qk, y = M)[name = tensor("op_1332")]; + tensor var_1333 = const()[name = tensor("op_1333"), val = tensor([1, 1])]; + tensor var_1334 = reshape(shape = var_1333, x = sqrt_s_t)[name = tensor("op_1334")]; + tensor M = real_div(x = var_1150, y = var_1334)[name = tensor("M")]; + tensor var_1336 = mul(x = qk, y = M)[name = tensor("op_1336")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_1314)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1332, y = v_17)[name = tensor("inner")]; - tensor var_1334_transpose_x_0 = const()[name = tensor("op_1334_transpose_x_0"), val = tensor(false)]; - tensor var_1334_transpose_y_0 = const()[name = tensor("op_1334_transpose_y_0"), val = tensor(false)]; - tensor var_1334 = matmul(transpose_x = var_1334_transpose_x_0, transpose_y = var_1334_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1334")]; - tensor var_1335 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1335")]; - tensor var_1336 = const()[name = tensor("op_1336"), val = tensor([1, 1, 1, 1])]; - tensor var_1337 = reshape(shape = var_1336, x = var_1335)[name = tensor("op_1337")]; - tensor cross = mul(x = var_1334, y = var_1337)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1318)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1336, y = v_17)[name = tensor("inner")]; + tensor var_1338_transpose_x_0 = const()[name = tensor("op_1338_transpose_x_0"), val = tensor(false)]; + tensor var_1338_transpose_y_0 = const()[name = tensor("op_1338_transpose_y_0"), val = tensor(false)]; + tensor var_1338 = matmul(transpose_x = var_1338_transpose_x_0, transpose_y = var_1338_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1338")]; + tensor var_1339 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1339")]; + tensor var_1340 = const()[name = tensor("op_1340"), val = tensor([1, 1, 1, 1])]; + tensor var_1341 = reshape(shape = var_1340, x = var_1339)[name = tensor("op_1341")]; + tensor cross = mul(x = var_1338, y = var_1341)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_1160)[name = tensor("v_masked")]; - tensor var_1343 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1343")]; - tensor var_1345_transpose_x_1 = const()[name = tensor("op_1345_transpose_x_1"), val = tensor(true)]; - tensor var_1345_transpose_y_1 = const()[name = tensor("op_1345_transpose_y_1"), val = tensor(false)]; - tensor var_1345 = matmul(transpose_x = var_1345_transpose_x_1, transpose_y = var_1345_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1345")]; - tensor new_kv_unnorm = add(x = var_1343, y = var_1345)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_1168)[name = tensor("new_scale")]; + tensor v_masked = mul(x = v_17, y = var_1164)[name = tensor("v_masked")]; + tensor var_1347 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1347")]; + tensor var_1349_transpose_x_1 = const()[name = tensor("op_1349_transpose_x_1"), val = tensor(true)]; + tensor var_1349_transpose_y_1 = const()[name = tensor("op_1349_transpose_y_1"), val = tensor(false)]; + tensor var_1349 = matmul(transpose_x = var_1349_transpose_x_1, transpose_y = var_1349_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1349")]; + tensor new_kv_unnorm = add(x = var_1347, y = var_1349)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1172)[name = tensor("new_scale")]; tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; - tensor clip_12 = clip(alpha = var_1026, beta = const_33, x = new_scale)[name = tensor("clip_12")]; + tensor clip_12 = clip(alpha = var_1029, beta = const_33, x = new_scale)[name = tensor("clip_12")]; tensor sqrt_new_scale = sqrt(x = clip_12)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_1354_perm_0 = const()[name = tensor("op_1354_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1358_perm_0 = const()[name = tensor("op_1358_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_1354 = transpose(perm = var_1354_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1023, x = var_1354)[name = tensor("out_33")]; - tensor var_1358 = const()[name = tensor("op_1358"), val = tensor([6, 1, 256])]; - tensor out = reshape(shape = var_1358, x = out_33)[name = tensor("out")]; - tensor var_1360 = silu(x = input_187)[name = tensor("op_1360")]; - tensor input_189 = mul(x = var_1360, y = out)[name = tensor("input_189")]; + tensor var_1358 = transpose(perm = var_1358_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1026, x = var_1358)[name = tensor("out_33")]; + tensor var_1362 = const()[name = tensor("op_1362"), val = tensor([6, 1, 256])]; + tensor out = reshape(shape = var_1362, x = out_33)[name = tensor("out")]; + tensor var_1364 = silu(x = input_187)[name = tensor("op_1364")]; + tensor input_189 = mul(x = var_1364, y = out)[name = tensor("input_189")]; tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1021, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; - tensor var_1370 = const()[name = tensor("op_1370"), val = tensor([1, 6, 1, 256])]; - tensor var_1371 = reshape(shape = var_1370, x = xt_5)[name = tensor("op_1371")]; - tensor var_1372_perm_0 = const()[name = tensor("op_1372_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1375 = const()[name = tensor("op_1375"), val = tensor([1, 6, 256])]; - tensor var_1372 = transpose(perm = var_1372_perm_0, x = var_1371)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_1375, x = var_1372)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1024, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1374 = const()[name = tensor("op_1374"), val = tensor([1, 6, 1, 256])]; + tensor var_1375 = reshape(shape = var_1374, x = xt_5)[name = tensor("op_1375")]; + tensor var_1376_perm_0 = const()[name = tensor("op_1376_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1379 = const()[name = tensor("op_1379"), val = tensor([1, 6, 256])]; + tensor var_1376 = transpose(perm = var_1376_perm_0, x = var_1375)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1379, x = var_1376)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_1398 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_5 = const()[name = tensor("concat_5"), val = tensor([6, 1, 3, 256])]; - tensor var_1400 = reshape(shape = concat_5, x = var_1398)[name = tensor("op_1400")]; - tensor var_1401_axes_0 = const()[name = tensor("op_1401_axes_0"), val = tensor([0])]; - tensor var_1401 = expand_dims(axes = var_1401_axes_0, x = var_1400)[name = tensor("op_1401")]; - tensor var_1402_perm_0 = const()[name = tensor("op_1402_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1403_axes_0 = const()[name = tensor("op_1403_axes_0"), val = tensor([-2])]; - tensor var_1402 = transpose(perm = var_1402_perm_0, x = var_1401)[name = tensor("transpose_8")]; - tensor var_1403 = squeeze(axes = var_1403_axes_0, x = var_1402)[name = tensor("op_1403")]; + tensor var_1402 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_6 = const()[name = tensor("concat_6"), val = tensor([6, 1, 3, 256])]; + tensor var_1404 = reshape(shape = concat_6, x = var_1402)[name = tensor("op_1404")]; + tensor var_1405_axes_0 = const()[name = tensor("op_1405_axes_0"), val = tensor([0])]; + tensor var_1405 = expand_dims(axes = var_1405_axes_0, x = var_1404)[name = tensor("op_1405")]; + tensor var_1406_perm_0 = const()[name = tensor("op_1406_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1407_axes_0 = const()[name = tensor("op_1407_axes_0"), val = tensor([-2])]; + tensor var_1406 = transpose(perm = var_1406_perm_0, x = var_1405)[name = tensor("transpose_8")]; + tensor var_1407 = squeeze(axes = var_1407_axes_0, x = var_1406)[name = tensor("op_1407")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 6, 1, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1403)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1407)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 6, 1, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1403)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1407)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 6, 1, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1403)[name = tensor("v_19")]; - tensor var_1411 = const()[name = tensor("op_1411"), val = tensor([6, 4, 64])]; - tensor var_1412 = reshape(shape = var_1411, x = q_19)[name = tensor("op_1412")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1407)[name = tensor("v_19")]; + tensor var_1415 = const()[name = tensor("op_1415"), val = tensor([6, 4, 64])]; + tensor var_1416 = reshape(shape = var_1415, x = q_19)[name = tensor("op_1416")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1418 = const()[name = tensor("op_1418"), val = tensor([6, 4, 64])]; - tensor var_1419 = reshape(shape = var_1418, x = k_19)[name = tensor("op_1419")]; + tensor var_1422 = const()[name = tensor("op_1422"), val = tensor([6, 4, 64])]; + tensor var_1423 = reshape(shape = var_1422, x = k_19)[name = tensor("op_1423")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1425 = const()[name = tensor("op_1425"), val = tensor([6, 4, 64])]; - tensor var_1426 = reshape(shape = var_1425, x = v_19)[name = tensor("op_1426")]; + tensor var_1429 = const()[name = tensor("op_1429"), val = tensor([6, 4, 64])]; + tensor var_1430 = reshape(shape = var_1429, x = v_19)[name = tensor("op_1430")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1429 = const()[name = tensor("op_1429"), val = tensor([1, 4, 6, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_1412)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_1429, x = q_21)[name = tensor("q")]; - tensor var_1431 = const()[name = tensor("op_1431"), val = tensor([1, 4, 6, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_1419)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_1431, x = k_21)[name = tensor("k")]; tensor var_1433 = const()[name = tensor("op_1433"), val = tensor([1, 4, 6, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_1426)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_1433, x = v_21)[name = tensor("v")]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1416)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1433, x = q_21)[name = tensor("q")]; + tensor var_1435 = const()[name = tensor("op_1435"), val = tensor([1, 4, 6, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1423)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1435, x = k_21)[name = tensor("k")]; + tensor var_1437 = const()[name = tensor("op_1437"), val = tensor([1, 4, 6, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1430)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1437, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -1253,35 +1262,35 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_1436 = const()[name = tensor("op_1436"), val = tensor([2, 0, 1, 3])]; - tensor var_1441 = const()[name = tensor("op_1441"), val = tensor([6, 256])]; - tensor var_1437 = transpose(perm = var_1436, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_1441, x = var_1437)[name = tensor("attn_output_11")]; + tensor var_1440 = const()[name = tensor("op_1440"), val = tensor([2, 0, 1, 3])]; + tensor var_1445 = const()[name = tensor("op_1445"), val = tensor([6, 256])]; + tensor var_1441 = transpose(perm = var_1440, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1445, x = var_1441)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_1445 = const()[name = tensor("op_1445"), val = tensor([6, 1, 256])]; - tensor attn_output = reshape(shape = var_1445, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1449 = const()[name = tensor("op_1449"), val = tensor([6, 1, 256])]; + tensor attn_output = reshape(shape = var_1449, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; - tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1021, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1024, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1021, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; - tensor var_1465 = const()[name = tensor("op_1465"), val = tensor([1, 1, 6, 256])]; - tensor input = reshape(shape = var_1465, x = xt)[name = tensor("input")]; - tensor var_1467 = const()[name = tensor("op_1467"), val = tensor([-1])]; - tensor var_1468 = reduce_l2_norm(axes = var_1467, keep_dims = var_1024, x = input)[name = tensor("op_1468")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1024, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1469 = const()[name = tensor("op_1469"), val = tensor([1, 1, 6, 256])]; + tensor input = reshape(shape = var_1469, x = xt)[name = tensor("input")]; + tensor var_1471 = const()[name = tensor("op_1471"), val = tensor([-1])]; + tensor var_1472 = reduce_l2_norm(axes = var_1471, keep_dims = var_1027, x = input)[name = tensor("op_1472")]; tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; - tensor clip_13 = clip(alpha = var_1015, beta = const_42, x = var_1468)[name = tensor("clip_13")]; - tensor var_1470 = real_div(x = input, y = clip_13)[name = tensor("op_1470")]; + tensor clip_13 = clip(alpha = var_1019, beta = const_42, x = var_1472)[name = tensor("clip_13")]; + tensor var_1474 = real_div(x = input, y = clip_13)[name = tensor("op_1474")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_10 = const()[name = tensor("concat_10"), val = tensor([1, 256, 6])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1470)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_10, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_11 = const()[name = tensor("concat_11"), val = tensor([1, 256, 6])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1474)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_11, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = emb, y = reshape_1)[name = tensor("matmul_2")]; @@ -1289,10 +1298,10 @@ program(1.0) tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 1, 5])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = matmul_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_1474")]; - tensor var_1476_axis_0 = const()[name = tensor("op_1476_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_1476_axis_0, values = (var_1172, nkv))[name = tensor("op_1476")]; - tensor var_1478_axis_0 = const()[name = tensor("op_1478_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_1478_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1478")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1478")]; + tensor var_1480_axis_0 = const()[name = tensor("op_1480_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1480_axis_0, values = (var_1176, nkv))[name = tensor("op_1480")]; + tensor var_1482_axis_0 = const()[name = tensor("op_1482_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1482_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1482")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/AMI/ls_eend_ami_100ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/AMI/ls_eend_ami_100ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index 8d182478f87d584d9d94476b7f2583b16717ad75..410c56363f92de5ad843b266d32a4135cbf90ff4 100644 --- a/optimized/AMI/ls_eend_ami_100ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/AMI/ls_eend_ami_100ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0ff551a62f8f5e57dc47abe2984bd72088f95d9c81c02509356f1020857d5ee2 -size 184719 +oid sha256:ab426518a231ac4eb22ca2bf826b391e45063a54f434ec9513622ea7bc91604f +size 185977 diff --git a/optimized/AMI/ls_eend_ami_100ms.mlpackage/Manifest.json b/optimized/AMI/ls_eend_ami_100ms.mlpackage/Manifest.json index b62b1fcc2b3f860b4dcfc3e90034d1ac1328f214..965f41685e7e60dd90a45054c89fc17bff41ef34 100644 --- a/optimized/AMI/ls_eend_ami_100ms.mlpackage/Manifest.json +++ b/optimized/AMI/ls_eend_ami_100ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "28D3AC02-8BCB-4DF7-BD59-D3E9848B3AD3": { + "BCD80537-45F4-46E8-BCAC-12FB7FBAAD92": { "author": "com.apple.CoreML", "description": "CoreML Model Weights", "name": "weights", "path": "com.apple.CoreML/weights" }, - "2DE470EA-5724-462D-ACC7-4F220EB2C67A": { + "DF12DDEE-40C4-4B47-823F-E8F351877440": { "author": "com.apple.CoreML", "description": "CoreML Model Specification", "name": "model.mlmodel", "path": "com.apple.CoreML/model.mlmodel" } }, - "rootModelIdentifier": "2DE470EA-5724-462D-ACC7-4F220EB2C67A" + "rootModelIdentifier": "DF12DDEE-40C4-4B47-823F-E8F351877440" } diff --git a/optimized/AMI/ls_eend_ami_200ms.mlmodelc/analytics/coremldata.bin b/optimized/AMI/ls_eend_ami_200ms.mlmodelc/analytics/coremldata.bin index 9d5f7ad1ba4e99a9d9c87e502b874807f6b522ab..d83da4a0f0fd1f992094f56d8c51d68d419ce76e 100644 --- a/optimized/AMI/ls_eend_ami_200ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/AMI/ls_eend_ami_200ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:04aef45fd0ca7469d16a407a1b240a052766eb185fe32c32543a38ea781af44f +oid sha256:9e24251c963d92a4cfcb32f900d3f1ce68671f28d5b3c59fd94ebc94429f0730 size 243 diff --git a/optimized/AMI/ls_eend_ami_200ms.mlmodelc/coremldata.bin b/optimized/AMI/ls_eend_ami_200ms.mlmodelc/coremldata.bin index a69838ab490cdcb129c4bee874c81e23ecdeb268..c6b50009475f2d3fc92376a7a42b90038b0d085a 100644 --- a/optimized/AMI/ls_eend_ami_200ms.mlmodelc/coremldata.bin +++ b/optimized/AMI/ls_eend_ami_200ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:880f0a3309d6d12838c629d2aa3a034badbe73a89f32d292371ee1a15990ebe5 -size 1250 +oid sha256:ba79bea1bc64a8cd704faa40f1c43140bcbfb3df3b0336a7db5cd015ed329011 +size 1341 diff --git a/optimized/AMI/ls_eend_ami_200ms.mlmodelc/metadata.json b/optimized/AMI/ls_eend_ami_200ms.mlmodelc/metadata.json index bcaba012d5caf68760abb25f13c9103dfa3c7ef5..933eae75437972764c0986a759c687edd31a6ed6 100644 --- a/optimized/AMI/ls_eend_ami_200ms.mlmodelc/metadata.json +++ b/optimized/AMI/ls_eend_ami_200ms.mlmodelc/metadata.json @@ -96,30 +96,30 @@ "Ios17.reshape" : 84, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 61, - "Split" : 8, + "Ios17.transpose" : 57, + "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 70, "Tile" : 9, - "Ios17.sliceByIndex" : 60, - "Ios16.sigmoid" : 9, + "Ios17.sliceByIndex" : 52, + "Ios16.sigmoid" : 5, "Ios16.reduceSum" : 1, "Ios17.squeeze" : 2, - "Ios17.layerNorm" : 37, - "Ios17.batchNorm" : 8, + "Ios17.layerNorm" : 33, + "Ios17.batchNorm" : 4, "Ios17.cast" : 16, "Ios17.sqrt" : 18, "Ios17.sub" : 16, - "Ios17.conv" : 25, + "Ios17.conv" : 13, "Ios17.realDiv" : 20, "Ios17.linear" : 56, "Ios17.concat" : 14, "Ios17.clip" : 22, "Ios17.greaterEqual" : 8, - "Ios16.silu" : 22, + "Ios16.silu" : 18, "Stack" : 6, "Ios16.cumsum" : 1, - "Ios17.mul" : 83 + "Ios17.mul" : 79 }, "computePrecision" : "Mixed (Float32, Int32)", "isUpdatable" : "0", @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"ami\", \"model_label\": \"AMI\", \"variant\": \"pipeline\", \"T\": 2, \"max_speakers\": 4, \"max_nspks\": 6, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"ami\", \"model_label\": \"AMI\", \"variant\": \"pipeline\", \"chunk_size\": 2, \"step_duration_ms\": 200, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 4, \"max_nspks\": 6, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/AMI/ls_eend_ami_200ms.mlmodelc/model.mil b/optimized/AMI/ls_eend_ami_200ms.mlmodelc/model.mil index 59a15c67c97757870660211fc0144cd52c36c00b..654ee4601cf092a29463c2ab1e6430af9833bdda 100644 --- a/optimized/AMI/ls_eend_ami_200ms.mlmodelc/model.mil +++ b/optimized/AMI/ls_eend_ami_200ms.mlmodelc/model.mil @@ -217,6 +217,7 @@ program(1.0) tensor var_21 = const()[name = tensor("op_21"), val = tensor(0x0p+0)]; tensor var_24 = const()[name = tensor("op_24"), val = tensor(0x1.0c6f7ap-20)]; tensor var_27 = const()[name = tensor("op_27"), val = tensor(2)]; + tensor var_30 = const()[name = tensor("op_30"), val = tensor(0)]; tensor var_33 = const()[name = tensor("op_33"), val = tensor(1)]; tensor var_35 = const()[name = tensor("op_35"), val = tensor(0x1.4f8b58p-17)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(true)]; @@ -263,8 +264,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_61")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_62")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_196 = const()[name = tensor("op_196"), val = tensor([2, 1])]; tensor var_197 = reshape(shape = var_196, x = sqrt_s_t_1)[name = tensor("op_197")]; @@ -272,7 +273,7 @@ program(1.0) tensor var_199 = mul(x = qk_1, y = M_1)[name = tensor("op_199")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_60")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_199, y = v_1)[name = tensor("inner_1")]; tensor var_201_transpose_x_0 = const()[name = tensor("op_201_transpose_x_0"), val = tensor(false)]; tensor var_201_transpose_y_0 = const()[name = tensor("op_201_transpose_y_0"), val = tensor(false)]; @@ -293,7 +294,7 @@ program(1.0) tensor var_214 = real_div(x = new_kv_unnorm_1, y = var_213)[name = tensor("op_214")]; tensor var_215_perm_0 = const()[name = tensor("op_215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_59")]; + tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_24, x = var_215)[name = tensor("out_3")]; tensor var_219 = const()[name = tensor("op_219"), val = tensor([1, 2, 256])]; tensor out_5 = reshape(shape = var_219, x = out_3)[name = tensor("out_5")]; @@ -341,145 +342,108 @@ program(1.0) tensor var_252 = sub(x = var_17, y = use_roll_1)[name = tensor("op_252")]; tensor var_253 = mul(x = filled_window_1, y = var_252)[name = tensor("op_253")]; tensor var_254 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_254")]; - tensor input_21 = add(x = var_253, y = var_254)[name = tensor("input_21")]; + tensor window_3 = add(x = var_253, y = var_254)[name = tensor("window_3")]; tensor var_256 = const()[name = tensor("op_256"), val = tensor(0x1p+0)]; tensor var_257 = add(x = filled_1, y = var_256)[name = tensor("op_257")]; tensor clip_1 = clip(alpha = var_21, beta = var_16, x = var_257)[name = tensor("clip_1")]; + tensor var_260_begin_0 = const()[name = tensor("op_260_begin_0"), val = tensor([0, 1, 0])]; + tensor var_260_end_0 = const()[name = tensor("op_260_end_0"), val = tensor([1, 1, 256])]; + tensor var_260_end_mask_0 = const()[name = tensor("op_260_end_mask_0"), val = tensor([true, true, true])]; + tensor var_260 = slice_by_index(begin = var_260_begin_0, end = var_260_end_0, end_mask = var_260_end_mask_0, x = x_3)[name = tensor("op_260")]; + tensor var_263_begin_0 = const()[name = tensor("op_263_begin_0"), val = tensor([0, 1, 0])]; + tensor var_263_end_0 = const()[name = tensor("op_263_end_0"), val = tensor([1, 16, 256])]; + tensor var_263_end_mask_0 = const()[name = tensor("op_263_end_mask_0"), val = tensor([true, true, true])]; + tensor var_263 = slice_by_index(begin = var_263_begin_0, end = var_263_end_0, end_mask = var_263_end_mask_0, x = window_3)[name = tensor("op_263")]; + tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; + tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_263, var_260))[name = tensor("rolled_3")]; + tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; + tensor var_268 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_268")]; + tensor var_269_dtype_0 = const()[name = tensor("op_269_dtype_0"), val = tensor("fp32")]; + tensor var_270 = const()[name = tensor("op_270"), val = tensor([1, 16, 1])]; + tensor var_269 = cast(dtype = var_269_dtype_0, x = var_268)[name = tensor("cast_141")]; + tensor fill_mask_3 = reshape(shape = var_270, x = var_269)[name = tensor("fill_mask_3")]; + tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_260)[name = tensor("x_expanded_3")]; + tensor var_274 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_274")]; + tensor var_275 = mul(x = window_3, y = var_274)[name = tensor("op_275")]; + tensor var_276 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_276")]; + tensor filled_window_3 = add(x = var_275, y = var_276)[name = tensor("filled_window_3")]; + tensor var_278 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_278")]; + tensor var_279_dtype_0 = const()[name = tensor("op_279_dtype_0"), val = tensor("fp32")]; + tensor var_280 = const()[name = tensor("op_280"), val = tensor([1, 1, 1])]; + tensor var_279 = cast(dtype = var_279_dtype_0, x = var_278)[name = tensor("cast_140")]; + tensor use_roll_3 = reshape(shape = var_280, x = var_279)[name = tensor("use_roll_3")]; + tensor var_282 = sub(x = var_17, y = use_roll_3)[name = tensor("op_282")]; + tensor var_283 = mul(x = filled_window_3, y = var_282)[name = tensor("op_283")]; + tensor var_284 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_284")]; + tensor window_5 = add(x = var_283, y = var_284)[name = tensor("window_5")]; + tensor var_286 = const()[name = tensor("op_286"), val = tensor(0x1p+0)]; + tensor var_287 = add(x = clip_1, y = var_286)[name = tensor("op_287")]; + tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_287)[name = tensor("clip_3")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_30, interleave = input_21_interleave_0, values = (window_3, window_5))[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; - tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; + tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; tensor inputs_3_pad_type_0 = const()[name = tensor("inputs_3_pad_type_0"), val = tensor("valid")]; tensor inputs_3_strides_0 = const()[name = tensor("inputs_3_strides_0"), val = tensor([1])]; tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_58")]; - tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_278_split_sizes_0 = const()[name = tensor("op_278_split_sizes_0"), val = tensor([256, 256])]; - tensor var_278_axis_0 = const()[name = tensor("op_278_axis_0"), val = tensor(1)]; - tensor var_278_0, tensor var_278_1 = split(axis = var_278_axis_0, split_sizes = var_278_split_sizes_0, x = inputs_3)[name = tensor("op_278")]; - tensor var_280 = sigmoid(x = var_278_1)[name = tensor("op_280")]; - tensor inputs_5 = mul(x = var_278_0, y = var_280)[name = tensor("inputs_5")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; + tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; + tensor var_310_split_sizes_0 = const()[name = tensor("op_310_split_sizes_0"), val = tensor([256, 256])]; + tensor var_310_axis_0 = const()[name = tensor("op_310_axis_0"), val = tensor(1)]; + tensor var_310_0, tensor var_310_1 = split(axis = var_310_axis_0, split_sizes = var_310_split_sizes_0, x = inputs_3)[name = tensor("op_310")]; + tensor var_312 = sigmoid(x = var_310_1)[name = tensor("op_312")]; + tensor inputs_5 = mul(x = var_310_0, y = var_312)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; tensor outputs_aug_1_strides_0 = const()[name = tensor("outputs_aug_1_strides_0"), val = tensor([1])]; tensor outputs_aug_1_dilations_0 = const()[name = tensor("outputs_aug_1_dilations_0"), val = tensor([1])]; - tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; + tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; tensor input_25_begin_0 = const()[name = tensor("input_25_begin_0"), val = tensor([0, 0, 0])]; - tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([1, 256, 16])]; + tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([2, 256, 16])]; tensor input_25_end_mask_0 = const()[name = tensor("input_25_end_mask_0"), val = tensor([true, true, false])]; - tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; - tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; - tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; + tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; + tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; + tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; tensor input_29_pad_type_0 = const()[name = tensor("input_29_pad_type_0"), val = tensor("valid")]; tensor input_29_strides_0 = const()[name = tensor("input_29_strides_0"), val = tensor([1])]; tensor input_29_pad_0 = const()[name = tensor("input_29_pad_0"), val = tensor([0, 0])]; tensor input_29_dilations_0 = const()[name = tensor("input_29_dilations_0"), val = tensor([1])]; tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; - tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; + tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_311_begin_0 = const()[name = tensor("op_311_begin_0"), val = tensor([0, -1, 0])]; - tensor var_311_end_0 = const()[name = tensor("op_311_end_0"), val = tensor([1, 16, 256])]; - tensor var_311_end_mask_0 = const()[name = tensor("op_311_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_57")]; - tensor var_311 = slice_by_index(begin = var_311_begin_0, end = var_311_end_0, end_mask = var_311_end_mask_0, x = conv_out_1)[name = tensor("op_311")]; - tensor var_314_begin_0 = const()[name = tensor("op_314_begin_0"), val = tensor([0, 1, 0])]; - tensor var_314_end_0 = const()[name = tensor("op_314_end_0"), val = tensor([1, 1, 256])]; - tensor var_314_end_mask_0 = const()[name = tensor("op_314_end_mask_0"), val = tensor([true, true, true])]; - tensor var_314 = slice_by_index(begin = var_314_begin_0, end = var_314_end_0, end_mask = var_314_end_mask_0, x = x_3)[name = tensor("op_314")]; - tensor var_317_begin_0 = const()[name = tensor("op_317_begin_0"), val = tensor([0, 1, 0])]; - tensor var_317_end_0 = const()[name = tensor("op_317_end_0"), val = tensor([1, 16, 256])]; - tensor var_317_end_mask_0 = const()[name = tensor("op_317_end_mask_0"), val = tensor([true, true, true])]; - tensor var_317 = slice_by_index(begin = var_317_begin_0, end = var_317_end_0, end_mask = var_317_end_mask_0, x = input_21)[name = tensor("op_317")]; - tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_317, var_314))[name = tensor("rolled_3")]; - tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; - tensor var_322 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_322")]; - tensor var_323_dtype_0 = const()[name = tensor("op_323_dtype_0"), val = tensor("fp32")]; - tensor var_324 = const()[name = tensor("op_324"), val = tensor([1, 16, 1])]; - tensor var_323 = cast(dtype = var_323_dtype_0, x = var_322)[name = tensor("cast_141")]; - tensor fill_mask_3 = reshape(shape = var_324, x = var_323)[name = tensor("fill_mask_3")]; - tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_314)[name = tensor("x_expanded_3")]; - tensor var_328 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_328")]; - tensor var_329 = mul(x = input_21, y = var_328)[name = tensor("op_329")]; - tensor var_330 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_330")]; - tensor filled_window_3 = add(x = var_329, y = var_330)[name = tensor("filled_window_3")]; - tensor var_332 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_332")]; - tensor var_333_dtype_0 = const()[name = tensor("op_333_dtype_0"), val = tensor("fp32")]; - tensor var_334 = const()[name = tensor("op_334"), val = tensor([1, 1, 1])]; - tensor var_333 = cast(dtype = var_333_dtype_0, x = var_332)[name = tensor("cast_140")]; - tensor use_roll_3 = reshape(shape = var_334, x = var_333)[name = tensor("use_roll_3")]; - tensor var_336 = sub(x = var_17, y = use_roll_3)[name = tensor("op_336")]; - tensor var_337 = mul(x = filled_window_3, y = var_336)[name = tensor("op_337")]; - tensor var_338 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_338")]; - tensor input_31 = add(x = var_337, y = var_338)[name = tensor("input_31")]; - tensor var_340 = const()[name = tensor("op_340"), val = tensor(0x1p+0)]; - tensor var_341 = add(x = clip_1, y = var_340)[name = tensor("op_341")]; - tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_341)[name = tensor("clip_3")]; - tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; - tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_31)[name = tensor("x_7")]; - tensor input_33_perm_0 = const()[name = tensor("input_33_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_9_pad_type_0 = const()[name = tensor("inputs_9_pad_type_0"), val = tensor("valid")]; - tensor inputs_9_strides_0 = const()[name = tensor("inputs_9_strides_0"), val = tensor([1])]; - tensor inputs_9_pad_0 = const()[name = tensor("inputs_9_pad_0"), val = tensor([0, 0])]; - tensor inputs_9_dilations_0 = const()[name = tensor("inputs_9_dilations_0"), val = tensor([1])]; - tensor inputs_9_groups_0 = const()[name = tensor("inputs_9_groups_0"), val = tensor(1)]; - tensor input_33 = transpose(perm = input_33_perm_0, x = x_7)[name = tensor("transpose_56")]; - tensor inputs_9 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_9_dilations_0, groups = inputs_9_groups_0, pad = inputs_9_pad_0, pad_type = inputs_9_pad_type_0, strides = inputs_9_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_33)[name = tensor("inputs_9")]; - tensor var_362_split_sizes_0 = const()[name = tensor("op_362_split_sizes_0"), val = tensor([256, 256])]; - tensor var_362_axis_0 = const()[name = tensor("op_362_axis_0"), val = tensor(1)]; - tensor var_362_0, tensor var_362_1 = split(axis = var_362_axis_0, split_sizes = var_362_split_sizes_0, x = inputs_9)[name = tensor("op_362")]; - tensor var_364 = sigmoid(x = var_362_1)[name = tensor("op_364")]; - tensor inputs_11 = mul(x = var_362_0, y = var_364)[name = tensor("inputs_11")]; - tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; - tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; - tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; - tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_11)[name = tensor("outputs_aug_3")]; - tensor input_35_begin_0 = const()[name = tensor("input_35_begin_0"), val = tensor([0, 0, 0])]; - tensor input_35_end_0 = const()[name = tensor("input_35_end_0"), val = tensor([1, 256, 16])]; - tensor input_35_end_mask_0 = const()[name = tensor("input_35_end_mask_0"), val = tensor([true, true, false])]; - tensor input_35 = slice_by_index(begin = input_35_begin_0, end = input_35_end_0, end_mask = input_35_end_mask_0, x = outputs_aug_3)[name = tensor("input_35")]; - tensor inputs_13 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_35)[name = tensor("inputs_13")]; - tensor input_37 = silu(x = inputs_13)[name = tensor("input_37")]; - tensor input_39_pad_type_0 = const()[name = tensor("input_39_pad_type_0"), val = tensor("valid")]; - tensor input_39_strides_0 = const()[name = tensor("input_39_strides_0"), val = tensor([1])]; - tensor input_39_pad_0 = const()[name = tensor("input_39_pad_0"), val = tensor([0, 0])]; - tensor input_39_dilations_0 = const()[name = tensor("input_39_dilations_0"), val = tensor([1])]; - tensor input_39_groups_0 = const()[name = tensor("input_39_groups_0"), val = tensor(1)]; - tensor input_39 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_37)[name = tensor("input_39")]; - tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_395_begin_0 = const()[name = tensor("op_395_begin_0"), val = tensor([0, -1, 0])]; - tensor var_395_end_0 = const()[name = tensor("op_395_end_0"), val = tensor([1, 16, 256])]; - tensor var_395_end_mask_0 = const()[name = tensor("op_395_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_39)[name = tensor("transpose_55")]; - tensor var_395 = slice_by_index(begin = var_395_begin_0, end = var_395_end_0, end_mask = var_395_end_mask_0, x = conv_out_3)[name = tensor("op_395")]; - tensor var_398_interleave_0 = const()[name = tensor("op_398_interleave_0"), val = tensor(false)]; - tensor var_398 = concat(axis = var_33, interleave = var_398_interleave_0, values = (var_311, var_395))[name = tensor("op_398")]; - tensor input_41 = add(x = x_3, y = var_398)[name = tensor("input_41")]; + tensor var_343_begin_0 = const()[name = tensor("op_343_begin_0"), val = tensor([0, -1, 0])]; + tensor var_343_end_0 = const()[name = tensor("op_343_end_0"), val = tensor([2, 16, 256])]; + tensor var_343_end_mask_0 = const()[name = tensor("op_343_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_343 = slice_by_index(begin = var_343_begin_0, end = var_343_end_0, end_mask = var_343_end_mask_0, x = conv_out_1)[name = tensor("op_343")]; + tensor var_345_perm_0 = const()[name = tensor("op_345_perm_0"), val = tensor([1, 0, 2])]; + tensor var_345 = transpose(perm = var_345_perm_0, x = var_343)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_345)[name = tensor("input_31")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; + tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; + tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; + tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; + tensor var_368 = const()[name = tensor("op_368"), val = tensor(0x1p-1)]; + tensor var_369 = mul(x = input_39, y = var_368)[name = tensor("op_369")]; + tensor input_41 = add(x = var_369, y = input_31)[name = tensor("input_41")]; tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; - tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_41)[name = tensor("input_43")]; - tensor inputs_15 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_43)[name = tensor("linear_8")]; - tensor input_45 = silu(x = inputs_15)[name = tensor("input_45")]; - tensor input_49 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_45)[name = tensor("linear_9")]; - tensor var_421 = const()[name = tensor("op_421"), val = tensor(0x1p-1)]; - tensor var_422 = mul(x = input_49, y = var_421)[name = tensor("op_422")]; - tensor input_51 = add(x = var_422, y = input_41)[name = tensor("input_51")]; - tensor input_53_axes_0 = const()[name = tensor("input_53_axes_0"), val = tensor([-1])]; - tensor input_53 = layer_norm(axes = input_53_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_51)[name = tensor("input_53")]; - tensor input_55_axes_0 = const()[name = tensor("input_55_axes_0"), val = tensor([-1])]; - tensor input_55 = layer_norm(axes = input_55_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_53)[name = tensor("input_55")]; - tensor inputs_17 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_55)[name = tensor("linear_10")]; - tensor input_57 = silu(x = inputs_17)[name = tensor("input_57")]; - tensor input_61 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_57)[name = tensor("linear_11")]; - tensor var_451 = const()[name = tensor("op_451"), val = tensor(0x1p-1)]; - tensor var_452 = mul(x = input_61, y = var_451)[name = tensor("op_452")]; - tensor input_63 = add(x = var_452, y = input_53)[name = tensor("input_63")]; - tensor x_9_axes_0 = const()[name = tensor("x_9_axes_0"), val = tensor([-1])]; - tensor x_9 = layer_norm(axes = x_9_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_63)[name = tensor("x_9")]; + tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; + tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; + tensor input_45 = layer_norm(axes = input_45_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_43)[name = tensor("input_45")]; + tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; + tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; + tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; + tensor var_398 = const()[name = tensor("op_398"), val = tensor(0x1p-1)]; + tensor var_399 = mul(x = input_51, y = var_398)[name = tensor("op_399")]; + tensor input_53 = add(x = var_399, y = input_43)[name = tensor("input_53")]; + tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; + tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_3_end_0 = const()[name = tensor("prev_kv_3_end_0"), val = tensor([2, 1, 4, 64, 64])]; tensor prev_kv_3_end_mask_0 = const()[name = tensor("prev_kv_3_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -490,243 +454,206 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_466 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_9)[name = tensor("linear_12")]; - tensor var_467 = const()[name = tensor("op_467"), val = tensor([1, 2, 4, 64])]; - tensor var_468 = reshape(shape = var_467, x = var_466)[name = tensor("op_468")]; + tensor var_413 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_414 = const()[name = tensor("op_414"), val = tensor([1, 2, 4, 64])]; + tensor var_415 = reshape(shape = var_414, x = var_413)[name = tensor("op_415")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_472 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_9)[name = tensor("linear_13")]; - tensor var_473 = const()[name = tensor("op_473"), val = tensor(0x1p-3)]; - tensor var_474 = mul(x = var_472, y = var_473)[name = tensor("op_474")]; - tensor var_475 = const()[name = tensor("op_475"), val = tensor([1, 2, 4, 64])]; - tensor var_476 = reshape(shape = var_475, x = var_474)[name = tensor("op_476")]; + tensor var_419 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_420 = const()[name = tensor("op_420"), val = tensor(0x1p-3)]; + tensor var_421 = mul(x = var_419, y = var_420)[name = tensor("op_421")]; + tensor var_422 = const()[name = tensor("op_422"), val = tensor([1, 2, 4, 64])]; + tensor var_423 = reshape(shape = var_422, x = var_421)[name = tensor("op_423")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_480 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_9)[name = tensor("linear_14")]; - tensor var_481 = const()[name = tensor("op_481"), val = tensor([1, 2, 4, 64])]; - tensor var_482 = reshape(shape = var_481, x = var_480)[name = tensor("op_482")]; + tensor var_427 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_428 = const()[name = tensor("op_428"), val = tensor([1, 2, 4, 64])]; + tensor var_429 = reshape(shape = var_428, x = var_427)[name = tensor("op_429")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_67 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_9)[name = tensor("linear_15")]; + tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; tensor s_t_3 = add(x = prev_scale_3, y = encoder__t_index)[name = tensor("s_t_3")]; tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_476)[name = tensor("transpose_53")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_468)[name = tensor("transpose_54")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_423)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_415)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_492 = const()[name = tensor("op_492"), val = tensor([2, 1])]; - tensor var_493 = reshape(shape = var_492, x = sqrt_s_t_3)[name = tensor("op_493")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_493)[name = tensor("M_3")]; - tensor var_495 = mul(x = qk_3, y = M_3)[name = tensor("op_495")]; + tensor var_439 = const()[name = tensor("op_439"), val = tensor([2, 1])]; + tensor var_440 = reshape(shape = var_439, x = sqrt_s_t_3)[name = tensor("op_440")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_440)[name = tensor("M_3")]; + tensor var_442 = mul(x = qk_3, y = M_3)[name = tensor("op_442")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_482)[name = tensor("transpose_52")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_495, y = v_3)[name = tensor("inner_3")]; - tensor var_497_transpose_x_0 = const()[name = tensor("op_497_transpose_x_0"), val = tensor(false)]; - tensor var_497_transpose_y_0 = const()[name = tensor("op_497_transpose_y_0"), val = tensor(false)]; - tensor var_497 = matmul(transpose_x = var_497_transpose_x_0, transpose_y = var_497_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_497")]; - tensor var_498 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_498")]; - tensor var_499 = const()[name = tensor("op_499"), val = tensor([1, 1, 2, 1])]; - tensor var_500 = reshape(shape = var_499, x = var_498)[name = tensor("op_500")]; - tensor cross_3 = mul(x = var_497, y = var_500)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_429)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_442, y = v_3)[name = tensor("inner_3")]; + tensor var_444_transpose_x_0 = const()[name = tensor("op_444_transpose_x_0"), val = tensor(false)]; + tensor var_444_transpose_y_0 = const()[name = tensor("op_444_transpose_y_0"), val = tensor(false)]; + tensor var_444 = matmul(transpose_x = var_444_transpose_x_0, transpose_y = var_444_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_444")]; + tensor var_445 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_445")]; + tensor var_446 = const()[name = tensor("op_446"), val = tensor([1, 1, 2, 1])]; + tensor var_447 = reshape(shape = var_446, x = var_445)[name = tensor("op_447")]; + tensor cross_3 = mul(x = var_444, y = var_447)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_503 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_503")]; - tensor var_505_transpose_x_1 = const()[name = tensor("op_505_transpose_x_1"), val = tensor(true)]; - tensor var_505_transpose_y_1 = const()[name = tensor("op_505_transpose_y_1"), val = tensor(false)]; - tensor var_505 = matmul(transpose_x = var_505_transpose_x_1, transpose_y = var_505_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_505")]; - tensor new_kv_unnorm_3 = add(x = var_503, y = var_505)[name = tensor("new_kv_unnorm_3")]; - tensor var_507 = const()[name = tensor("op_507"), val = tensor(0x1p+1)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_507)[name = tensor("new_scale_3")]; - tensor var_509 = sqrt(x = new_scale_3)[name = tensor("op_509")]; - tensor var_510 = real_div(x = new_kv_unnorm_3, y = var_509)[name = tensor("op_510")]; - tensor var_511_perm_0 = const()[name = tensor("op_511_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_450 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_450")]; + tensor var_452_transpose_x_1 = const()[name = tensor("op_452_transpose_x_1"), val = tensor(true)]; + tensor var_452_transpose_y_1 = const()[name = tensor("op_452_transpose_y_1"), val = tensor(false)]; + tensor var_452 = matmul(transpose_x = var_452_transpose_x_1, transpose_y = var_452_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_452")]; + tensor new_kv_unnorm_3 = add(x = var_450, y = var_452)[name = tensor("new_kv_unnorm_3")]; + tensor var_454 = const()[name = tensor("op_454"), val = tensor(0x1p+1)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_454)[name = tensor("new_scale_3")]; + tensor var_456 = sqrt(x = new_scale_3)[name = tensor("op_456")]; + tensor var_457 = real_div(x = new_kv_unnorm_3, y = var_456)[name = tensor("op_457")]; + tensor var_458_perm_0 = const()[name = tensor("op_458_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_511 = transpose(perm = var_511_perm_0, x = out_7)[name = tensor("transpose_51")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_511)[name = tensor("out_9")]; - tensor var_515 = const()[name = tensor("op_515"), val = tensor([1, 2, 256])]; - tensor out_11 = reshape(shape = var_515, x = out_9)[name = tensor("out_11")]; - tensor var_517 = silu(x = input_67)[name = tensor("op_517")]; - tensor input_69 = mul(x = var_517, y = out_11)[name = tensor("input_69")]; - tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_69)[name = tensor("linear_16")]; - tensor x_11 = add(x = input_63, y = ret_out_3)[name = tensor("x_11")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor var_458 = transpose(perm = var_458_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_458)[name = tensor("out_9")]; + tensor var_462 = const()[name = tensor("op_462"), val = tensor([1, 2, 256])]; + tensor out_11 = reshape(shape = var_462, x = out_9)[name = tensor("out_11")]; + tensor var_464 = silu(x = input_57)[name = tensor("op_464")]; + tensor input_59 = mul(x = var_464, y = out_11)[name = tensor("input_59")]; + tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; + tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; + tensor window_7_begin_0 = const()[name = tensor("window_7_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_7_end_0 = const()[name = tensor("window_7_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_7_end_mask_0 = const()[name = tensor("window_7_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_7_squeeze_mask_0 = const()[name = tensor("window_7_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_7 = slice_by_index(begin = window_7_begin_0, end = window_7_end_0, end_mask = window_7_end_mask_0, squeeze_mask = window_7_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_7")]; tensor filled_5_begin_0 = const()[name = tensor("filled_5_begin_0"), val = tensor([1, 0])]; tensor filled_5_end_0 = const()[name = tensor("filled_5_end_0"), val = tensor([2, 1])]; tensor filled_5_end_mask_0 = const()[name = tensor("filled_5_end_mask_0"), val = tensor([false, true])]; tensor filled_5_squeeze_mask_0 = const()[name = tensor("filled_5_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_5 = slice_by_index(begin = filled_5_begin_0, end = filled_5_end_0, end_mask = filled_5_end_mask_0, squeeze_mask = filled_5_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_5")]; - tensor var_526_begin_0 = const()[name = tensor("op_526_begin_0"), val = tensor([0, 0, 0])]; - tensor var_526_end_0 = const()[name = tensor("op_526_end_0"), val = tensor([1, 1, 256])]; - tensor var_526_end_mask_0 = const()[name = tensor("op_526_end_mask_0"), val = tensor([true, false, true])]; - tensor var_526 = slice_by_index(begin = var_526_begin_0, end = var_526_end_0, end_mask = var_526_end_mask_0, x = x_11)[name = tensor("op_526")]; - tensor var_529_begin_0 = const()[name = tensor("op_529_begin_0"), val = tensor([0, 1, 0])]; - tensor var_529_end_0 = const()[name = tensor("op_529_end_0"), val = tensor([1, 16, 256])]; - tensor var_529_end_mask_0 = const()[name = tensor("op_529_end_mask_0"), val = tensor([true, true, true])]; - tensor var_529 = slice_by_index(begin = var_529_begin_0, end = var_529_end_0, end_mask = var_529_end_mask_0, x = window_3)[name = tensor("op_529")]; + tensor var_473_begin_0 = const()[name = tensor("op_473_begin_0"), val = tensor([0, 0, 0])]; + tensor var_473_end_0 = const()[name = tensor("op_473_end_0"), val = tensor([1, 1, 256])]; + tensor var_473_end_mask_0 = const()[name = tensor("op_473_end_mask_0"), val = tensor([true, false, true])]; + tensor var_473 = slice_by_index(begin = var_473_begin_0, end = var_473_end_0, end_mask = var_473_end_mask_0, x = x_9)[name = tensor("op_473")]; + tensor var_476_begin_0 = const()[name = tensor("op_476_begin_0"), val = tensor([0, 1, 0])]; + tensor var_476_end_0 = const()[name = tensor("op_476_end_0"), val = tensor([1, 16, 256])]; + tensor var_476_end_mask_0 = const()[name = tensor("op_476_end_mask_0"), val = tensor([true, true, true])]; + tensor var_476 = slice_by_index(begin = var_476_begin_0, end = var_476_end_0, end_mask = var_476_end_mask_0, x = window_7)[name = tensor("op_476")]; tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_529, var_526))[name = tensor("rolled_5")]; + tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_476, var_473))[name = tensor("rolled_5")]; tensor clip_4 = clip(alpha = var_21, beta = var_20, x = filled_5)[name = tensor("clip_4")]; - tensor var_534 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_534")]; - tensor var_535_dtype_0 = const()[name = tensor("op_535_dtype_0"), val = tensor("fp32")]; - tensor var_536 = const()[name = tensor("op_536"), val = tensor([1, 16, 1])]; - tensor var_535 = cast(dtype = var_535_dtype_0, x = var_534)[name = tensor("cast_139")]; - tensor fill_mask_5 = reshape(shape = var_536, x = var_535)[name = tensor("fill_mask_5")]; + tensor var_481 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_481")]; + tensor var_482_dtype_0 = const()[name = tensor("op_482_dtype_0"), val = tensor("fp32")]; + tensor var_483 = const()[name = tensor("op_483"), val = tensor([1, 16, 1])]; + tensor var_482 = cast(dtype = var_482_dtype_0, x = var_481)[name = tensor("cast_139")]; + tensor fill_mask_5 = reshape(shape = var_483, x = var_482)[name = tensor("fill_mask_5")]; tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_526)[name = tensor("x_expanded_5")]; - tensor var_540 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_540")]; - tensor var_541 = mul(x = window_3, y = var_540)[name = tensor("op_541")]; - tensor var_542 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_542")]; - tensor filled_window_5 = add(x = var_541, y = var_542)[name = tensor("filled_window_5")]; - tensor var_544 = greater_equal(x = filled_5, y = var_16)[name = tensor("op_544")]; - tensor var_545_dtype_0 = const()[name = tensor("op_545_dtype_0"), val = tensor("fp32")]; - tensor var_546 = const()[name = tensor("op_546"), val = tensor([1, 1, 1])]; - tensor var_545 = cast(dtype = var_545_dtype_0, x = var_544)[name = tensor("cast_138")]; - tensor use_roll_5 = reshape(shape = var_546, x = var_545)[name = tensor("use_roll_5")]; - tensor var_548 = sub(x = var_17, y = use_roll_5)[name = tensor("op_548")]; - tensor var_549 = mul(x = filled_window_5, y = var_548)[name = tensor("op_549")]; - tensor var_550 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_550")]; - tensor input_71 = add(x = var_549, y = var_550)[name = tensor("input_71")]; - tensor var_552 = const()[name = tensor("op_552"), val = tensor(0x1p+0)]; - tensor var_553 = add(x = filled_5, y = var_552)[name = tensor("op_553")]; - tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_553)[name = tensor("clip_5")]; - tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; - tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_71)[name = tensor("x_13")]; - tensor input_73_perm_0 = const()[name = tensor("input_73_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_19_pad_type_0 = const()[name = tensor("inputs_19_pad_type_0"), val = tensor("valid")]; - tensor inputs_19_strides_0 = const()[name = tensor("inputs_19_strides_0"), val = tensor([1])]; - tensor inputs_19_pad_0 = const()[name = tensor("inputs_19_pad_0"), val = tensor([0, 0])]; - tensor inputs_19_dilations_0 = const()[name = tensor("inputs_19_dilations_0"), val = tensor([1])]; - tensor inputs_19_groups_0 = const()[name = tensor("inputs_19_groups_0"), val = tensor(1)]; - tensor input_73 = transpose(perm = input_73_perm_0, x = x_13)[name = tensor("transpose_50")]; - tensor inputs_19 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_19_dilations_0, groups = inputs_19_groups_0, pad = inputs_19_pad_0, pad_type = inputs_19_pad_type_0, strides = inputs_19_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_73)[name = tensor("inputs_19")]; - tensor var_574_split_sizes_0 = const()[name = tensor("op_574_split_sizes_0"), val = tensor([256, 256])]; - tensor var_574_axis_0 = const()[name = tensor("op_574_axis_0"), val = tensor(1)]; - tensor var_574_0, tensor var_574_1 = split(axis = var_574_axis_0, split_sizes = var_574_split_sizes_0, x = inputs_19)[name = tensor("op_574")]; - tensor var_576 = sigmoid(x = var_574_1)[name = tensor("op_576")]; - tensor inputs_21 = mul(x = var_574_0, y = var_576)[name = tensor("inputs_21")]; - tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; - tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; - tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; - tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_21)[name = tensor("outputs_aug_5")]; - tensor input_75_begin_0 = const()[name = tensor("input_75_begin_0"), val = tensor([0, 0, 0])]; - tensor input_75_end_0 = const()[name = tensor("input_75_end_0"), val = tensor([1, 256, 16])]; - tensor input_75_end_mask_0 = const()[name = tensor("input_75_end_mask_0"), val = tensor([true, true, false])]; - tensor input_75 = slice_by_index(begin = input_75_begin_0, end = input_75_end_0, end_mask = input_75_end_mask_0, x = outputs_aug_5)[name = tensor("input_75")]; - tensor inputs_23 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_75)[name = tensor("inputs_23")]; - tensor input_77 = silu(x = inputs_23)[name = tensor("input_77")]; - tensor input_79_pad_type_0 = const()[name = tensor("input_79_pad_type_0"), val = tensor("valid")]; - tensor input_79_strides_0 = const()[name = tensor("input_79_strides_0"), val = tensor([1])]; - tensor input_79_pad_0 = const()[name = tensor("input_79_pad_0"), val = tensor([0, 0])]; - tensor input_79_dilations_0 = const()[name = tensor("input_79_dilations_0"), val = tensor([1])]; - tensor input_79_groups_0 = const()[name = tensor("input_79_groups_0"), val = tensor(1)]; - tensor input_79 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_79_dilations_0, groups = input_79_groups_0, pad = input_79_pad_0, pad_type = input_79_pad_type_0, strides = input_79_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_77)[name = tensor("input_79")]; - tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_607_begin_0 = const()[name = tensor("op_607_begin_0"), val = tensor([0, -1, 0])]; - tensor var_607_end_0 = const()[name = tensor("op_607_end_0"), val = tensor([1, 16, 256])]; - tensor var_607_end_mask_0 = const()[name = tensor("op_607_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_79)[name = tensor("transpose_49")]; - tensor var_607 = slice_by_index(begin = var_607_begin_0, end = var_607_end_0, end_mask = var_607_end_mask_0, x = conv_out_5)[name = tensor("op_607")]; - tensor var_610_begin_0 = const()[name = tensor("op_610_begin_0"), val = tensor([0, 1, 0])]; - tensor var_610_end_0 = const()[name = tensor("op_610_end_0"), val = tensor([1, 1, 256])]; - tensor var_610_end_mask_0 = const()[name = tensor("op_610_end_mask_0"), val = tensor([true, true, true])]; - tensor var_610 = slice_by_index(begin = var_610_begin_0, end = var_610_end_0, end_mask = var_610_end_mask_0, x = x_11)[name = tensor("op_610")]; - tensor var_613_begin_0 = const()[name = tensor("op_613_begin_0"), val = tensor([0, 1, 0])]; - tensor var_613_end_0 = const()[name = tensor("op_613_end_0"), val = tensor([1, 16, 256])]; - tensor var_613_end_mask_0 = const()[name = tensor("op_613_end_mask_0"), val = tensor([true, true, true])]; - tensor var_613 = slice_by_index(begin = var_613_begin_0, end = var_613_end_0, end_mask = var_613_end_mask_0, x = input_71)[name = tensor("op_613")]; + tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_473)[name = tensor("x_expanded_5")]; + tensor var_487 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_487")]; + tensor var_488 = mul(x = window_7, y = var_487)[name = tensor("op_488")]; + tensor var_489 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_489")]; + tensor filled_window_5 = add(x = var_488, y = var_489)[name = tensor("filled_window_5")]; + tensor var_491 = greater_equal(x = filled_5, y = var_16)[name = tensor("op_491")]; + tensor var_492_dtype_0 = const()[name = tensor("op_492_dtype_0"), val = tensor("fp32")]; + tensor var_493 = const()[name = tensor("op_493"), val = tensor([1, 1, 1])]; + tensor var_492 = cast(dtype = var_492_dtype_0, x = var_491)[name = tensor("cast_138")]; + tensor use_roll_5 = reshape(shape = var_493, x = var_492)[name = tensor("use_roll_5")]; + tensor var_495 = sub(x = var_17, y = use_roll_5)[name = tensor("op_495")]; + tensor var_496 = mul(x = filled_window_5, y = var_495)[name = tensor("op_496")]; + tensor var_497 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_497")]; + tensor window_9 = add(x = var_496, y = var_497)[name = tensor("window_9")]; + tensor var_499 = const()[name = tensor("op_499"), val = tensor(0x1p+0)]; + tensor var_500 = add(x = filled_5, y = var_499)[name = tensor("op_500")]; + tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_500)[name = tensor("clip_5")]; + tensor var_503_begin_0 = const()[name = tensor("op_503_begin_0"), val = tensor([0, 1, 0])]; + tensor var_503_end_0 = const()[name = tensor("op_503_end_0"), val = tensor([1, 1, 256])]; + tensor var_503_end_mask_0 = const()[name = tensor("op_503_end_mask_0"), val = tensor([true, true, true])]; + tensor var_503 = slice_by_index(begin = var_503_begin_0, end = var_503_end_0, end_mask = var_503_end_mask_0, x = x_9)[name = tensor("op_503")]; + tensor var_506_begin_0 = const()[name = tensor("op_506_begin_0"), val = tensor([0, 1, 0])]; + tensor var_506_end_0 = const()[name = tensor("op_506_end_0"), val = tensor([1, 16, 256])]; + tensor var_506_end_mask_0 = const()[name = tensor("op_506_end_mask_0"), val = tensor([true, true, true])]; + tensor var_506 = slice_by_index(begin = var_506_begin_0, end = var_506_end_0, end_mask = var_506_end_mask_0, x = window_9)[name = tensor("op_506")]; tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; - tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_613, var_610))[name = tensor("rolled_7")]; + tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_506, var_503))[name = tensor("rolled_7")]; tensor clip_6 = clip(alpha = var_21, beta = var_20, x = clip_5)[name = tensor("clip_6")]; - tensor var_618 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_618")]; - tensor var_619_dtype_0 = const()[name = tensor("op_619_dtype_0"), val = tensor("fp32")]; - tensor var_620 = const()[name = tensor("op_620"), val = tensor([1, 16, 1])]; - tensor var_619 = cast(dtype = var_619_dtype_0, x = var_618)[name = tensor("cast_137")]; - tensor fill_mask_7 = reshape(shape = var_620, x = var_619)[name = tensor("fill_mask_7")]; + tensor var_511 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_511")]; + tensor var_512_dtype_0 = const()[name = tensor("op_512_dtype_0"), val = tensor("fp32")]; + tensor var_513 = const()[name = tensor("op_513"), val = tensor([1, 16, 1])]; + tensor var_512 = cast(dtype = var_512_dtype_0, x = var_511)[name = tensor("cast_137")]; + tensor fill_mask_7 = reshape(shape = var_513, x = var_512)[name = tensor("fill_mask_7")]; tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_610)[name = tensor("x_expanded_7")]; - tensor var_624 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_624")]; - tensor var_625 = mul(x = input_71, y = var_624)[name = tensor("op_625")]; - tensor var_626 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_626")]; - tensor filled_window_7 = add(x = var_625, y = var_626)[name = tensor("filled_window_7")]; - tensor var_628 = greater_equal(x = clip_5, y = var_16)[name = tensor("op_628")]; - tensor var_629_dtype_0 = const()[name = tensor("op_629_dtype_0"), val = tensor("fp32")]; - tensor var_630 = const()[name = tensor("op_630"), val = tensor([1, 1, 1])]; - tensor var_629 = cast(dtype = var_629_dtype_0, x = var_628)[name = tensor("cast_136")]; - tensor use_roll_7 = reshape(shape = var_630, x = var_629)[name = tensor("use_roll_7")]; - tensor var_632 = sub(x = var_17, y = use_roll_7)[name = tensor("op_632")]; - tensor var_633 = mul(x = filled_window_7, y = var_632)[name = tensor("op_633")]; - tensor var_634 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_634")]; - tensor input_81 = add(x = var_633, y = var_634)[name = tensor("input_81")]; - tensor var_636 = const()[name = tensor("op_636"), val = tensor(0x1p+0)]; - tensor var_637 = add(x = clip_5, y = var_636)[name = tensor("op_637")]; - tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_637)[name = tensor("clip_7")]; - tensor x_15_axes_0 = const()[name = tensor("x_15_axes_0"), val = tensor([-1])]; - tensor x_15 = layer_norm(axes = x_15_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_81)[name = tensor("x_15")]; - tensor input_83_perm_0 = const()[name = tensor("input_83_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_25_pad_type_0 = const()[name = tensor("inputs_25_pad_type_0"), val = tensor("valid")]; - tensor inputs_25_strides_0 = const()[name = tensor("inputs_25_strides_0"), val = tensor([1])]; - tensor inputs_25_pad_0 = const()[name = tensor("inputs_25_pad_0"), val = tensor([0, 0])]; - tensor inputs_25_dilations_0 = const()[name = tensor("inputs_25_dilations_0"), val = tensor([1])]; - tensor inputs_25_groups_0 = const()[name = tensor("inputs_25_groups_0"), val = tensor(1)]; - tensor input_83 = transpose(perm = input_83_perm_0, x = x_15)[name = tensor("transpose_48")]; - tensor inputs_25 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_25_dilations_0, groups = inputs_25_groups_0, pad = inputs_25_pad_0, pad_type = inputs_25_pad_type_0, strides = inputs_25_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_83)[name = tensor("inputs_25")]; - tensor var_658_split_sizes_0 = const()[name = tensor("op_658_split_sizes_0"), val = tensor([256, 256])]; - tensor var_658_axis_0 = const()[name = tensor("op_658_axis_0"), val = tensor(1)]; - tensor var_658_0, tensor var_658_1 = split(axis = var_658_axis_0, split_sizes = var_658_split_sizes_0, x = inputs_25)[name = tensor("op_658")]; - tensor var_660 = sigmoid(x = var_658_1)[name = tensor("op_660")]; - tensor inputs_27 = mul(x = var_658_0, y = var_660)[name = tensor("inputs_27")]; - tensor outputs_aug_7_pad_type_0 = const()[name = tensor("outputs_aug_7_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_7_pad_0 = const()[name = tensor("outputs_aug_7_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_7_groups_0 = const()[name = tensor("outputs_aug_7_groups_0"), val = tensor(256)]; - tensor outputs_aug_7_strides_0 = const()[name = tensor("outputs_aug_7_strides_0"), val = tensor([1])]; - tensor outputs_aug_7_dilations_0 = const()[name = tensor("outputs_aug_7_dilations_0"), val = tensor([1])]; - tensor outputs_aug_7 = conv(dilations = outputs_aug_7_dilations_0, groups = outputs_aug_7_groups_0, pad = outputs_aug_7_pad_0, pad_type = outputs_aug_7_pad_type_0, strides = outputs_aug_7_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_27)[name = tensor("outputs_aug_7")]; - tensor input_85_begin_0 = const()[name = tensor("input_85_begin_0"), val = tensor([0, 0, 0])]; - tensor input_85_end_0 = const()[name = tensor("input_85_end_0"), val = tensor([1, 256, 16])]; - tensor input_85_end_mask_0 = const()[name = tensor("input_85_end_mask_0"), val = tensor([true, true, false])]; - tensor input_85 = slice_by_index(begin = input_85_begin_0, end = input_85_end_0, end_mask = input_85_end_mask_0, x = outputs_aug_7)[name = tensor("input_85")]; - tensor inputs_29 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_85)[name = tensor("inputs_29")]; - tensor input_87 = silu(x = inputs_29)[name = tensor("input_87")]; - tensor input_89_pad_type_0 = const()[name = tensor("input_89_pad_type_0"), val = tensor("valid")]; - tensor input_89_strides_0 = const()[name = tensor("input_89_strides_0"), val = tensor([1])]; - tensor input_89_pad_0 = const()[name = tensor("input_89_pad_0"), val = tensor([0, 0])]; - tensor input_89_dilations_0 = const()[name = tensor("input_89_dilations_0"), val = tensor([1])]; - tensor input_89_groups_0 = const()[name = tensor("input_89_groups_0"), val = tensor(1)]; - tensor input_89 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_89_dilations_0, groups = input_89_groups_0, pad = input_89_pad_0, pad_type = input_89_pad_type_0, strides = input_89_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_87)[name = tensor("input_89")]; - tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_691_begin_0 = const()[name = tensor("op_691_begin_0"), val = tensor([0, -1, 0])]; - tensor var_691_end_0 = const()[name = tensor("op_691_end_0"), val = tensor([1, 16, 256])]; - tensor var_691_end_mask_0 = const()[name = tensor("op_691_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_89)[name = tensor("transpose_47")]; - tensor var_691 = slice_by_index(begin = var_691_begin_0, end = var_691_end_0, end_mask = var_691_end_mask_0, x = conv_out_7)[name = tensor("op_691")]; - tensor var_694_interleave_0 = const()[name = tensor("op_694_interleave_0"), val = tensor(false)]; - tensor var_694 = concat(axis = var_33, interleave = var_694_interleave_0, values = (var_607, var_691))[name = tensor("op_694")]; - tensor input_91 = add(x = x_11, y = var_694)[name = tensor("input_91")]; - tensor input_93_axes_0 = const()[name = tensor("input_93_axes_0"), val = tensor([-1])]; - tensor input_93 = layer_norm(axes = input_93_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_91)[name = tensor("input_93")]; - tensor inputs_31 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_93)[name = tensor("linear_17")]; - tensor input_95 = silu(x = inputs_31)[name = tensor("input_95")]; - tensor input_99 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_95)[name = tensor("linear_18")]; - tensor var_717 = const()[name = tensor("op_717"), val = tensor(0x1p-1)]; - tensor var_718 = mul(x = input_99, y = var_717)[name = tensor("op_718")]; - tensor input_101 = add(x = var_718, y = input_91)[name = tensor("input_101")]; - tensor input_103_axes_0 = const()[name = tensor("input_103_axes_0"), val = tensor([-1])]; - tensor input_103 = layer_norm(axes = input_103_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_101)[name = tensor("input_103")]; - tensor input_105_axes_0 = const()[name = tensor("input_105_axes_0"), val = tensor([-1])]; - tensor input_105 = layer_norm(axes = input_105_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_103)[name = tensor("input_105")]; - tensor inputs_33 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_105)[name = tensor("linear_19")]; - tensor input_107 = silu(x = inputs_33)[name = tensor("input_107")]; - tensor input_111 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_107)[name = tensor("linear_20")]; - tensor var_747 = const()[name = tensor("op_747"), val = tensor(0x1p-1)]; - tensor var_748 = mul(x = input_111, y = var_747)[name = tensor("op_748")]; - tensor input_113 = add(x = var_748, y = input_103)[name = tensor("input_113")]; - tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; - tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_113)[name = tensor("x_17")]; + tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_503)[name = tensor("x_expanded_7")]; + tensor var_517 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_517")]; + tensor var_518 = mul(x = window_9, y = var_517)[name = tensor("op_518")]; + tensor var_519 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_519")]; + tensor filled_window_7 = add(x = var_518, y = var_519)[name = tensor("filled_window_7")]; + tensor var_521 = greater_equal(x = clip_5, y = var_16)[name = tensor("op_521")]; + tensor var_522_dtype_0 = const()[name = tensor("op_522_dtype_0"), val = tensor("fp32")]; + tensor var_523 = const()[name = tensor("op_523"), val = tensor([1, 1, 1])]; + tensor var_522 = cast(dtype = var_522_dtype_0, x = var_521)[name = tensor("cast_136")]; + tensor use_roll_7 = reshape(shape = var_523, x = var_522)[name = tensor("use_roll_7")]; + tensor var_525 = sub(x = var_17, y = use_roll_7)[name = tensor("op_525")]; + tensor var_526 = mul(x = filled_window_7, y = var_525)[name = tensor("op_526")]; + tensor var_527 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_527")]; + tensor window_11 = add(x = var_526, y = var_527)[name = tensor("window_11")]; + tensor var_529 = const()[name = tensor("op_529"), val = tensor(0x1p+0)]; + tensor var_530 = add(x = clip_5, y = var_529)[name = tensor("op_530")]; + tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_530)[name = tensor("clip_7")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_30, interleave = input_61_interleave_0, values = (window_9, window_11))[name = tensor("input_61")]; + tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; + tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; + tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_13_pad_type_0 = const()[name = tensor("inputs_13_pad_type_0"), val = tensor("valid")]; + tensor inputs_13_strides_0 = const()[name = tensor("inputs_13_strides_0"), val = tensor([1])]; + tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; + tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; + tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; + tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; + tensor var_553_split_sizes_0 = const()[name = tensor("op_553_split_sizes_0"), val = tensor([256, 256])]; + tensor var_553_axis_0 = const()[name = tensor("op_553_axis_0"), val = tensor(1)]; + tensor var_553_0, tensor var_553_1 = split(axis = var_553_axis_0, split_sizes = var_553_split_sizes_0, x = inputs_13)[name = tensor("op_553")]; + tensor var_555 = sigmoid(x = var_553_1)[name = tensor("op_555")]; + tensor inputs_15 = mul(x = var_553_0, y = var_555)[name = tensor("inputs_15")]; + tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; + tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; + tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; + tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_15)[name = tensor("outputs_aug_3")]; + tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; + tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([2, 256, 16])]; + tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; + tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_3)[name = tensor("input_65")]; + tensor inputs_17 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_65)[name = tensor("inputs_17")]; + tensor input_67 = silu(x = inputs_17)[name = tensor("input_67")]; + tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; + tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; + tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; + tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; + tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; + tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; + tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; + tensor var_586_begin_0 = const()[name = tensor("op_586_begin_0"), val = tensor([0, -1, 0])]; + tensor var_586_end_0 = const()[name = tensor("op_586_end_0"), val = tensor([2, 16, 256])]; + tensor var_586_end_mask_0 = const()[name = tensor("op_586_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_586 = slice_by_index(begin = var_586_begin_0, end = var_586_end_0, end_mask = var_586_end_mask_0, x = conv_out_3)[name = tensor("op_586")]; + tensor var_588_perm_0 = const()[name = tensor("op_588_perm_0"), val = tensor([1, 0, 2])]; + tensor var_588 = transpose(perm = var_588_perm_0, x = var_586)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_588)[name = tensor("input_71")]; + tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; + tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; + tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; + tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; + tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; + tensor var_611 = const()[name = tensor("op_611"), val = tensor(0x1p-1)]; + tensor var_612 = mul(x = input_79, y = var_611)[name = tensor("op_612")]; + tensor input_81 = add(x = var_612, y = input_71)[name = tensor("input_81")]; + tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; + tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; + tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; + tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; + tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; + tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; + tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; + tensor var_641 = const()[name = tensor("op_641"), val = tensor(0x1p-1)]; + tensor var_642 = mul(x = input_91, y = var_641)[name = tensor("op_642")]; + tensor input_93 = add(x = var_642, y = input_83)[name = tensor("input_93")]; + tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; + tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; tensor prev_kv_5_end_0 = const()[name = tensor("prev_kv_5_end_0"), val = tensor([3, 1, 4, 64, 64])]; tensor prev_kv_5_end_mask_0 = const()[name = tensor("prev_kv_5_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -737,243 +664,206 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_762 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_17)[name = tensor("linear_21")]; - tensor var_763 = const()[name = tensor("op_763"), val = tensor([1, 2, 4, 64])]; - tensor var_764 = reshape(shape = var_763, x = var_762)[name = tensor("op_764")]; + tensor var_656 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_657 = const()[name = tensor("op_657"), val = tensor([1, 2, 4, 64])]; + tensor var_658 = reshape(shape = var_657, x = var_656)[name = tensor("op_658")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_768 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_17)[name = tensor("linear_22")]; - tensor var_769 = const()[name = tensor("op_769"), val = tensor(0x1p-3)]; - tensor var_770 = mul(x = var_768, y = var_769)[name = tensor("op_770")]; - tensor var_771 = const()[name = tensor("op_771"), val = tensor([1, 2, 4, 64])]; - tensor var_772 = reshape(shape = var_771, x = var_770)[name = tensor("op_772")]; + tensor var_662 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_663 = const()[name = tensor("op_663"), val = tensor(0x1p-3)]; + tensor var_664 = mul(x = var_662, y = var_663)[name = tensor("op_664")]; + tensor var_665 = const()[name = tensor("op_665"), val = tensor([1, 2, 4, 64])]; + tensor var_666 = reshape(shape = var_665, x = var_664)[name = tensor("op_666")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_776 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_17)[name = tensor("linear_23")]; - tensor var_777 = const()[name = tensor("op_777"), val = tensor([1, 2, 4, 64])]; - tensor var_778 = reshape(shape = var_777, x = var_776)[name = tensor("op_778")]; + tensor var_670 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_671 = const()[name = tensor("op_671"), val = tensor([1, 2, 4, 64])]; + tensor var_672 = reshape(shape = var_671, x = var_670)[name = tensor("op_672")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_117 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_17)[name = tensor("linear_24")]; + tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; tensor s_t_5 = add(x = prev_scale_5, y = encoder__t_index)[name = tensor("s_t_5")]; tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_772)[name = tensor("transpose_45")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_764)[name = tensor("transpose_46")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_666)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_658)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_788 = const()[name = tensor("op_788"), val = tensor([2, 1])]; - tensor var_789 = reshape(shape = var_788, x = sqrt_s_t_5)[name = tensor("op_789")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_789)[name = tensor("M_5")]; - tensor var_791 = mul(x = qk_5, y = M_5)[name = tensor("op_791")]; + tensor var_682 = const()[name = tensor("op_682"), val = tensor([2, 1])]; + tensor var_683 = reshape(shape = var_682, x = sqrt_s_t_5)[name = tensor("op_683")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_683)[name = tensor("M_5")]; + tensor var_685 = mul(x = qk_5, y = M_5)[name = tensor("op_685")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_778)[name = tensor("transpose_44")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_791, y = v_5)[name = tensor("inner_5")]; - tensor var_793_transpose_x_0 = const()[name = tensor("op_793_transpose_x_0"), val = tensor(false)]; - tensor var_793_transpose_y_0 = const()[name = tensor("op_793_transpose_y_0"), val = tensor(false)]; - tensor var_793 = matmul(transpose_x = var_793_transpose_x_0, transpose_y = var_793_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_793")]; - tensor var_794 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_794")]; - tensor var_795 = const()[name = tensor("op_795"), val = tensor([1, 1, 2, 1])]; - tensor var_796 = reshape(shape = var_795, x = var_794)[name = tensor("op_796")]; - tensor cross_5 = mul(x = var_793, y = var_796)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_672)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_685, y = v_5)[name = tensor("inner_5")]; + tensor var_687_transpose_x_0 = const()[name = tensor("op_687_transpose_x_0"), val = tensor(false)]; + tensor var_687_transpose_y_0 = const()[name = tensor("op_687_transpose_y_0"), val = tensor(false)]; + tensor var_687 = matmul(transpose_x = var_687_transpose_x_0, transpose_y = var_687_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_687")]; + tensor var_688 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_688")]; + tensor var_689 = const()[name = tensor("op_689"), val = tensor([1, 1, 2, 1])]; + tensor var_690 = reshape(shape = var_689, x = var_688)[name = tensor("op_690")]; + tensor cross_5 = mul(x = var_687, y = var_690)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_799 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_799")]; - tensor var_801_transpose_x_1 = const()[name = tensor("op_801_transpose_x_1"), val = tensor(true)]; - tensor var_801_transpose_y_1 = const()[name = tensor("op_801_transpose_y_1"), val = tensor(false)]; - tensor var_801 = matmul(transpose_x = var_801_transpose_x_1, transpose_y = var_801_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_801")]; - tensor new_kv_unnorm_5 = add(x = var_799, y = var_801)[name = tensor("new_kv_unnorm_5")]; - tensor var_803 = const()[name = tensor("op_803"), val = tensor(0x1p+1)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_803)[name = tensor("new_scale_5")]; - tensor var_805 = sqrt(x = new_scale_5)[name = tensor("op_805")]; - tensor var_806 = real_div(x = new_kv_unnorm_5, y = var_805)[name = tensor("op_806")]; - tensor var_807_perm_0 = const()[name = tensor("op_807_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_693 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_693")]; + tensor var_695_transpose_x_1 = const()[name = tensor("op_695_transpose_x_1"), val = tensor(true)]; + tensor var_695_transpose_y_1 = const()[name = tensor("op_695_transpose_y_1"), val = tensor(false)]; + tensor var_695 = matmul(transpose_x = var_695_transpose_x_1, transpose_y = var_695_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_695")]; + tensor new_kv_unnorm_5 = add(x = var_693, y = var_695)[name = tensor("new_kv_unnorm_5")]; + tensor var_697 = const()[name = tensor("op_697"), val = tensor(0x1p+1)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_697)[name = tensor("new_scale_5")]; + tensor var_699 = sqrt(x = new_scale_5)[name = tensor("op_699")]; + tensor var_700 = real_div(x = new_kv_unnorm_5, y = var_699)[name = tensor("op_700")]; + tensor var_701_perm_0 = const()[name = tensor("op_701_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_807 = transpose(perm = var_807_perm_0, x = out_13)[name = tensor("transpose_43")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_807)[name = tensor("out_15")]; - tensor var_811 = const()[name = tensor("op_811"), val = tensor([1, 2, 256])]; - tensor out_17 = reshape(shape = var_811, x = out_15)[name = tensor("out_17")]; - tensor var_813 = silu(x = input_117)[name = tensor("op_813")]; - tensor input_119 = mul(x = var_813, y = out_17)[name = tensor("input_119")]; - tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_119)[name = tensor("linear_25")]; - tensor x_19 = add(x = input_113, y = ret_out_5)[name = tensor("x_19")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor var_701 = transpose(perm = var_701_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_701)[name = tensor("out_15")]; + tensor var_705 = const()[name = tensor("op_705"), val = tensor([1, 2, 256])]; + tensor out_17 = reshape(shape = var_705, x = out_15)[name = tensor("out_17")]; + tensor var_707 = silu(x = input_97)[name = tensor("op_707")]; + tensor input_99 = mul(x = var_707, y = out_17)[name = tensor("input_99")]; + tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; + tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; + tensor window_13_begin_0 = const()[name = tensor("window_13_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_13_end_0 = const()[name = tensor("window_13_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_13_end_mask_0 = const()[name = tensor("window_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_13_squeeze_mask_0 = const()[name = tensor("window_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_13 = slice_by_index(begin = window_13_begin_0, end = window_13_end_0, end_mask = window_13_end_mask_0, squeeze_mask = window_13_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_13")]; tensor filled_9_begin_0 = const()[name = tensor("filled_9_begin_0"), val = tensor([2, 0])]; tensor filled_9_end_0 = const()[name = tensor("filled_9_end_0"), val = tensor([3, 1])]; tensor filled_9_end_mask_0 = const()[name = tensor("filled_9_end_mask_0"), val = tensor([false, true])]; tensor filled_9_squeeze_mask_0 = const()[name = tensor("filled_9_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_9 = slice_by_index(begin = filled_9_begin_0, end = filled_9_end_0, end_mask = filled_9_end_mask_0, squeeze_mask = filled_9_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_9")]; - tensor var_822_begin_0 = const()[name = tensor("op_822_begin_0"), val = tensor([0, 0, 0])]; - tensor var_822_end_0 = const()[name = tensor("op_822_end_0"), val = tensor([1, 1, 256])]; - tensor var_822_end_mask_0 = const()[name = tensor("op_822_end_mask_0"), val = tensor([true, false, true])]; - tensor var_822 = slice_by_index(begin = var_822_begin_0, end = var_822_end_0, end_mask = var_822_end_mask_0, x = x_19)[name = tensor("op_822")]; - tensor var_825_begin_0 = const()[name = tensor("op_825_begin_0"), val = tensor([0, 1, 0])]; - tensor var_825_end_0 = const()[name = tensor("op_825_end_0"), val = tensor([1, 16, 256])]; - tensor var_825_end_mask_0 = const()[name = tensor("op_825_end_mask_0"), val = tensor([true, true, true])]; - tensor var_825 = slice_by_index(begin = var_825_begin_0, end = var_825_end_0, end_mask = var_825_end_mask_0, x = window_5)[name = tensor("op_825")]; + tensor var_716_begin_0 = const()[name = tensor("op_716_begin_0"), val = tensor([0, 0, 0])]; + tensor var_716_end_0 = const()[name = tensor("op_716_end_0"), val = tensor([1, 1, 256])]; + tensor var_716_end_mask_0 = const()[name = tensor("op_716_end_mask_0"), val = tensor([true, false, true])]; + tensor var_716 = slice_by_index(begin = var_716_begin_0, end = var_716_end_0, end_mask = var_716_end_mask_0, x = x_15)[name = tensor("op_716")]; + tensor var_719_begin_0 = const()[name = tensor("op_719_begin_0"), val = tensor([0, 1, 0])]; + tensor var_719_end_0 = const()[name = tensor("op_719_end_0"), val = tensor([1, 16, 256])]; + tensor var_719_end_mask_0 = const()[name = tensor("op_719_end_mask_0"), val = tensor([true, true, true])]; + tensor var_719 = slice_by_index(begin = var_719_begin_0, end = var_719_end_0, end_mask = var_719_end_mask_0, x = window_13)[name = tensor("op_719")]; tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; - tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_825, var_822))[name = tensor("rolled_9")]; + tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_719, var_716))[name = tensor("rolled_9")]; tensor clip_8 = clip(alpha = var_21, beta = var_20, x = filled_9)[name = tensor("clip_8")]; - tensor var_830 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_830")]; - tensor var_831_dtype_0 = const()[name = tensor("op_831_dtype_0"), val = tensor("fp32")]; - tensor var_832 = const()[name = tensor("op_832"), val = tensor([1, 16, 1])]; - tensor var_831 = cast(dtype = var_831_dtype_0, x = var_830)[name = tensor("cast_135")]; - tensor fill_mask_9 = reshape(shape = var_832, x = var_831)[name = tensor("fill_mask_9")]; + tensor var_724 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_724")]; + tensor var_725_dtype_0 = const()[name = tensor("op_725_dtype_0"), val = tensor("fp32")]; + tensor var_726 = const()[name = tensor("op_726"), val = tensor([1, 16, 1])]; + tensor var_725 = cast(dtype = var_725_dtype_0, x = var_724)[name = tensor("cast_135")]; + tensor fill_mask_9 = reshape(shape = var_726, x = var_725)[name = tensor("fill_mask_9")]; tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_822)[name = tensor("x_expanded_9")]; - tensor var_836 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_836")]; - tensor var_837 = mul(x = window_5, y = var_836)[name = tensor("op_837")]; - tensor var_838 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_838")]; - tensor filled_window_9 = add(x = var_837, y = var_838)[name = tensor("filled_window_9")]; - tensor var_840 = greater_equal(x = filled_9, y = var_16)[name = tensor("op_840")]; - tensor var_841_dtype_0 = const()[name = tensor("op_841_dtype_0"), val = tensor("fp32")]; - tensor var_842 = const()[name = tensor("op_842"), val = tensor([1, 1, 1])]; - tensor var_841 = cast(dtype = var_841_dtype_0, x = var_840)[name = tensor("cast_134")]; - tensor use_roll_9 = reshape(shape = var_842, x = var_841)[name = tensor("use_roll_9")]; - tensor var_844 = sub(x = var_17, y = use_roll_9)[name = tensor("op_844")]; - tensor var_845 = mul(x = filled_window_9, y = var_844)[name = tensor("op_845")]; - tensor var_846 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_846")]; - tensor input_121 = add(x = var_845, y = var_846)[name = tensor("input_121")]; - tensor var_848 = const()[name = tensor("op_848"), val = tensor(0x1p+0)]; - tensor var_849 = add(x = filled_9, y = var_848)[name = tensor("op_849")]; - tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_849)[name = tensor("clip_9")]; - tensor x_21_axes_0 = const()[name = tensor("x_21_axes_0"), val = tensor([-1])]; - tensor x_21 = layer_norm(axes = x_21_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_121)[name = tensor("x_21")]; - tensor input_123_perm_0 = const()[name = tensor("input_123_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_35_pad_type_0 = const()[name = tensor("inputs_35_pad_type_0"), val = tensor("valid")]; - tensor inputs_35_strides_0 = const()[name = tensor("inputs_35_strides_0"), val = tensor([1])]; - tensor inputs_35_pad_0 = const()[name = tensor("inputs_35_pad_0"), val = tensor([0, 0])]; - tensor inputs_35_dilations_0 = const()[name = tensor("inputs_35_dilations_0"), val = tensor([1])]; - tensor inputs_35_groups_0 = const()[name = tensor("inputs_35_groups_0"), val = tensor(1)]; - tensor input_123 = transpose(perm = input_123_perm_0, x = x_21)[name = tensor("transpose_42")]; - tensor inputs_35 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_35_dilations_0, groups = inputs_35_groups_0, pad = inputs_35_pad_0, pad_type = inputs_35_pad_type_0, strides = inputs_35_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_123)[name = tensor("inputs_35")]; - tensor var_870_split_sizes_0 = const()[name = tensor("op_870_split_sizes_0"), val = tensor([256, 256])]; - tensor var_870_axis_0 = const()[name = tensor("op_870_axis_0"), val = tensor(1)]; - tensor var_870_0, tensor var_870_1 = split(axis = var_870_axis_0, split_sizes = var_870_split_sizes_0, x = inputs_35)[name = tensor("op_870")]; - tensor var_872 = sigmoid(x = var_870_1)[name = tensor("op_872")]; - tensor inputs_37 = mul(x = var_870_0, y = var_872)[name = tensor("inputs_37")]; - tensor outputs_aug_9_pad_type_0 = const()[name = tensor("outputs_aug_9_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_9_pad_0 = const()[name = tensor("outputs_aug_9_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_9_groups_0 = const()[name = tensor("outputs_aug_9_groups_0"), val = tensor(256)]; - tensor outputs_aug_9_strides_0 = const()[name = tensor("outputs_aug_9_strides_0"), val = tensor([1])]; - tensor outputs_aug_9_dilations_0 = const()[name = tensor("outputs_aug_9_dilations_0"), val = tensor([1])]; - tensor outputs_aug_9 = conv(dilations = outputs_aug_9_dilations_0, groups = outputs_aug_9_groups_0, pad = outputs_aug_9_pad_0, pad_type = outputs_aug_9_pad_type_0, strides = outputs_aug_9_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_37)[name = tensor("outputs_aug_9")]; - tensor input_125_begin_0 = const()[name = tensor("input_125_begin_0"), val = tensor([0, 0, 0])]; - tensor input_125_end_0 = const()[name = tensor("input_125_end_0"), val = tensor([1, 256, 16])]; - tensor input_125_end_mask_0 = const()[name = tensor("input_125_end_mask_0"), val = tensor([true, true, false])]; - tensor input_125 = slice_by_index(begin = input_125_begin_0, end = input_125_end_0, end_mask = input_125_end_mask_0, x = outputs_aug_9)[name = tensor("input_125")]; - tensor inputs_39 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_125)[name = tensor("inputs_39")]; - tensor input_127 = silu(x = inputs_39)[name = tensor("input_127")]; - tensor input_129_pad_type_0 = const()[name = tensor("input_129_pad_type_0"), val = tensor("valid")]; - tensor input_129_strides_0 = const()[name = tensor("input_129_strides_0"), val = tensor([1])]; - tensor input_129_pad_0 = const()[name = tensor("input_129_pad_0"), val = tensor([0, 0])]; - tensor input_129_dilations_0 = const()[name = tensor("input_129_dilations_0"), val = tensor([1])]; - tensor input_129_groups_0 = const()[name = tensor("input_129_groups_0"), val = tensor(1)]; - tensor input_129 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_129_dilations_0, groups = input_129_groups_0, pad = input_129_pad_0, pad_type = input_129_pad_type_0, strides = input_129_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_127)[name = tensor("input_129")]; - tensor conv_out_9_perm_0 = const()[name = tensor("conv_out_9_perm_0"), val = tensor([0, 2, 1])]; - tensor var_903_begin_0 = const()[name = tensor("op_903_begin_0"), val = tensor([0, -1, 0])]; - tensor var_903_end_0 = const()[name = tensor("op_903_end_0"), val = tensor([1, 16, 256])]; - tensor var_903_end_mask_0 = const()[name = tensor("op_903_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_9 = transpose(perm = conv_out_9_perm_0, x = input_129)[name = tensor("transpose_41")]; - tensor var_903 = slice_by_index(begin = var_903_begin_0, end = var_903_end_0, end_mask = var_903_end_mask_0, x = conv_out_9)[name = tensor("op_903")]; - tensor var_906_begin_0 = const()[name = tensor("op_906_begin_0"), val = tensor([0, 1, 0])]; - tensor var_906_end_0 = const()[name = tensor("op_906_end_0"), val = tensor([1, 1, 256])]; - tensor var_906_end_mask_0 = const()[name = tensor("op_906_end_mask_0"), val = tensor([true, true, true])]; - tensor var_906 = slice_by_index(begin = var_906_begin_0, end = var_906_end_0, end_mask = var_906_end_mask_0, x = x_19)[name = tensor("op_906")]; - tensor var_909_begin_0 = const()[name = tensor("op_909_begin_0"), val = tensor([0, 1, 0])]; - tensor var_909_end_0 = const()[name = tensor("op_909_end_0"), val = tensor([1, 16, 256])]; - tensor var_909_end_mask_0 = const()[name = tensor("op_909_end_mask_0"), val = tensor([true, true, true])]; - tensor var_909 = slice_by_index(begin = var_909_begin_0, end = var_909_end_0, end_mask = var_909_end_mask_0, x = input_121)[name = tensor("op_909")]; + tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_716)[name = tensor("x_expanded_9")]; + tensor var_730 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_730")]; + tensor var_731 = mul(x = window_13, y = var_730)[name = tensor("op_731")]; + tensor var_732 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_732")]; + tensor filled_window_9 = add(x = var_731, y = var_732)[name = tensor("filled_window_9")]; + tensor var_734 = greater_equal(x = filled_9, y = var_16)[name = tensor("op_734")]; + tensor var_735_dtype_0 = const()[name = tensor("op_735_dtype_0"), val = tensor("fp32")]; + tensor var_736 = const()[name = tensor("op_736"), val = tensor([1, 1, 1])]; + tensor var_735 = cast(dtype = var_735_dtype_0, x = var_734)[name = tensor("cast_134")]; + tensor use_roll_9 = reshape(shape = var_736, x = var_735)[name = tensor("use_roll_9")]; + tensor var_738 = sub(x = var_17, y = use_roll_9)[name = tensor("op_738")]; + tensor var_739 = mul(x = filled_window_9, y = var_738)[name = tensor("op_739")]; + tensor var_740 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_740")]; + tensor window_15 = add(x = var_739, y = var_740)[name = tensor("window_15")]; + tensor var_742 = const()[name = tensor("op_742"), val = tensor(0x1p+0)]; + tensor var_743 = add(x = filled_9, y = var_742)[name = tensor("op_743")]; + tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_743)[name = tensor("clip_9")]; + tensor var_746_begin_0 = const()[name = tensor("op_746_begin_0"), val = tensor([0, 1, 0])]; + tensor var_746_end_0 = const()[name = tensor("op_746_end_0"), val = tensor([1, 1, 256])]; + tensor var_746_end_mask_0 = const()[name = tensor("op_746_end_mask_0"), val = tensor([true, true, true])]; + tensor var_746 = slice_by_index(begin = var_746_begin_0, end = var_746_end_0, end_mask = var_746_end_mask_0, x = x_15)[name = tensor("op_746")]; + tensor var_749_begin_0 = const()[name = tensor("op_749_begin_0"), val = tensor([0, 1, 0])]; + tensor var_749_end_0 = const()[name = tensor("op_749_end_0"), val = tensor([1, 16, 256])]; + tensor var_749_end_mask_0 = const()[name = tensor("op_749_end_mask_0"), val = tensor([true, true, true])]; + tensor var_749 = slice_by_index(begin = var_749_begin_0, end = var_749_end_0, end_mask = var_749_end_mask_0, x = window_15)[name = tensor("op_749")]; tensor rolled_11_interleave_0 = const()[name = tensor("rolled_11_interleave_0"), val = tensor(false)]; - tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_909, var_906))[name = tensor("rolled_11")]; + tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_749, var_746))[name = tensor("rolled_11")]; tensor clip_10 = clip(alpha = var_21, beta = var_20, x = clip_9)[name = tensor("clip_10")]; - tensor var_914 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_914")]; - tensor var_915_dtype_0 = const()[name = tensor("op_915_dtype_0"), val = tensor("fp32")]; - tensor var_916 = const()[name = tensor("op_916"), val = tensor([1, 16, 1])]; - tensor var_915 = cast(dtype = var_915_dtype_0, x = var_914)[name = tensor("cast_133")]; - tensor fill_mask_11 = reshape(shape = var_916, x = var_915)[name = tensor("fill_mask_11")]; + tensor var_754 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_754")]; + tensor var_755_dtype_0 = const()[name = tensor("op_755_dtype_0"), val = tensor("fp32")]; + tensor var_756 = const()[name = tensor("op_756"), val = tensor([1, 16, 1])]; + tensor var_755 = cast(dtype = var_755_dtype_0, x = var_754)[name = tensor("cast_133")]; + tensor fill_mask_11 = reshape(shape = var_756, x = var_755)[name = tensor("fill_mask_11")]; tensor x_expanded_11_reps_0 = const()[name = tensor("x_expanded_11_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_906)[name = tensor("x_expanded_11")]; - tensor var_920 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_920")]; - tensor var_921 = mul(x = input_121, y = var_920)[name = tensor("op_921")]; - tensor var_922 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_922")]; - tensor filled_window_11 = add(x = var_921, y = var_922)[name = tensor("filled_window_11")]; - tensor var_924 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_924")]; - tensor var_925_dtype_0 = const()[name = tensor("op_925_dtype_0"), val = tensor("fp32")]; - tensor var_926 = const()[name = tensor("op_926"), val = tensor([1, 1, 1])]; - tensor var_925 = cast(dtype = var_925_dtype_0, x = var_924)[name = tensor("cast_132")]; - tensor use_roll_11 = reshape(shape = var_926, x = var_925)[name = tensor("use_roll_11")]; - tensor var_928 = sub(x = var_17, y = use_roll_11)[name = tensor("op_928")]; - tensor var_929 = mul(x = filled_window_11, y = var_928)[name = tensor("op_929")]; - tensor var_930 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_930")]; - tensor input_131 = add(x = var_929, y = var_930)[name = tensor("input_131")]; - tensor var_932 = const()[name = tensor("op_932"), val = tensor(0x1p+0)]; - tensor var_933 = add(x = clip_9, y = var_932)[name = tensor("op_933")]; - tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_933)[name = tensor("clip_11")]; - tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; - tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_131)[name = tensor("x_23")]; - tensor input_133_perm_0 = const()[name = tensor("input_133_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_41_pad_type_0 = const()[name = tensor("inputs_41_pad_type_0"), val = tensor("valid")]; - tensor inputs_41_strides_0 = const()[name = tensor("inputs_41_strides_0"), val = tensor([1])]; - tensor inputs_41_pad_0 = const()[name = tensor("inputs_41_pad_0"), val = tensor([0, 0])]; - tensor inputs_41_dilations_0 = const()[name = tensor("inputs_41_dilations_0"), val = tensor([1])]; - tensor inputs_41_groups_0 = const()[name = tensor("inputs_41_groups_0"), val = tensor(1)]; - tensor input_133 = transpose(perm = input_133_perm_0, x = x_23)[name = tensor("transpose_40")]; - tensor inputs_41 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_41_dilations_0, groups = inputs_41_groups_0, pad = inputs_41_pad_0, pad_type = inputs_41_pad_type_0, strides = inputs_41_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_133)[name = tensor("inputs_41")]; - tensor var_954_split_sizes_0 = const()[name = tensor("op_954_split_sizes_0"), val = tensor([256, 256])]; - tensor var_954_axis_0 = const()[name = tensor("op_954_axis_0"), val = tensor(1)]; - tensor var_954_0, tensor var_954_1 = split(axis = var_954_axis_0, split_sizes = var_954_split_sizes_0, x = inputs_41)[name = tensor("op_954")]; - tensor var_956 = sigmoid(x = var_954_1)[name = tensor("op_956")]; - tensor inputs_43 = mul(x = var_954_0, y = var_956)[name = tensor("inputs_43")]; - tensor outputs_aug_11_pad_type_0 = const()[name = tensor("outputs_aug_11_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_11_pad_0 = const()[name = tensor("outputs_aug_11_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_11_groups_0 = const()[name = tensor("outputs_aug_11_groups_0"), val = tensor(256)]; - tensor outputs_aug_11_strides_0 = const()[name = tensor("outputs_aug_11_strides_0"), val = tensor([1])]; - tensor outputs_aug_11_dilations_0 = const()[name = tensor("outputs_aug_11_dilations_0"), val = tensor([1])]; - tensor outputs_aug_11 = conv(dilations = outputs_aug_11_dilations_0, groups = outputs_aug_11_groups_0, pad = outputs_aug_11_pad_0, pad_type = outputs_aug_11_pad_type_0, strides = outputs_aug_11_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_43)[name = tensor("outputs_aug_11")]; - tensor input_135_begin_0 = const()[name = tensor("input_135_begin_0"), val = tensor([0, 0, 0])]; - tensor input_135_end_0 = const()[name = tensor("input_135_end_0"), val = tensor([1, 256, 16])]; - tensor input_135_end_mask_0 = const()[name = tensor("input_135_end_mask_0"), val = tensor([true, true, false])]; - tensor input_135 = slice_by_index(begin = input_135_begin_0, end = input_135_end_0, end_mask = input_135_end_mask_0, x = outputs_aug_11)[name = tensor("input_135")]; - tensor inputs_45 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_135)[name = tensor("inputs_45")]; - tensor input_137 = silu(x = inputs_45)[name = tensor("input_137")]; - tensor input_139_pad_type_0 = const()[name = tensor("input_139_pad_type_0"), val = tensor("valid")]; - tensor input_139_strides_0 = const()[name = tensor("input_139_strides_0"), val = tensor([1])]; - tensor input_139_pad_0 = const()[name = tensor("input_139_pad_0"), val = tensor([0, 0])]; - tensor input_139_dilations_0 = const()[name = tensor("input_139_dilations_0"), val = tensor([1])]; - tensor input_139_groups_0 = const()[name = tensor("input_139_groups_0"), val = tensor(1)]; - tensor input_139 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_139_dilations_0, groups = input_139_groups_0, pad = input_139_pad_0, pad_type = input_139_pad_type_0, strides = input_139_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_137)[name = tensor("input_139")]; - tensor conv_out_11_perm_0 = const()[name = tensor("conv_out_11_perm_0"), val = tensor([0, 2, 1])]; - tensor var_987_begin_0 = const()[name = tensor("op_987_begin_0"), val = tensor([0, -1, 0])]; - tensor var_987_end_0 = const()[name = tensor("op_987_end_0"), val = tensor([1, 16, 256])]; - tensor var_987_end_mask_0 = const()[name = tensor("op_987_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_11 = transpose(perm = conv_out_11_perm_0, x = input_139)[name = tensor("transpose_39")]; - tensor var_987 = slice_by_index(begin = var_987_begin_0, end = var_987_end_0, end_mask = var_987_end_mask_0, x = conv_out_11)[name = tensor("op_987")]; - tensor var_990_interleave_0 = const()[name = tensor("op_990_interleave_0"), val = tensor(false)]; - tensor var_990 = concat(axis = var_33, interleave = var_990_interleave_0, values = (var_903, var_987))[name = tensor("op_990")]; - tensor input_141 = add(x = x_19, y = var_990)[name = tensor("input_141")]; - tensor input_143_axes_0 = const()[name = tensor("input_143_axes_0"), val = tensor([-1])]; - tensor input_143 = layer_norm(axes = input_143_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_141)[name = tensor("input_143")]; - tensor inputs_47 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_143)[name = tensor("linear_26")]; - tensor input_145 = silu(x = inputs_47)[name = tensor("input_145")]; - tensor input_149 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_145)[name = tensor("linear_27")]; - tensor var_1013 = const()[name = tensor("op_1013"), val = tensor(0x1p-1)]; - tensor var_1014 = mul(x = input_149, y = var_1013)[name = tensor("op_1014")]; - tensor input_151 = add(x = var_1014, y = input_141)[name = tensor("input_151")]; - tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; - tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_151)[name = tensor("input_153")]; - tensor input_155_axes_0 = const()[name = tensor("input_155_axes_0"), val = tensor([-1])]; - tensor input_155 = layer_norm(axes = input_155_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_153)[name = tensor("input_155")]; - tensor inputs_49 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_155)[name = tensor("linear_28")]; - tensor input_157 = silu(x = inputs_49)[name = tensor("input_157")]; - tensor input_161 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_157)[name = tensor("linear_29")]; - tensor var_1043 = const()[name = tensor("op_1043"), val = tensor(0x1p-1)]; - tensor var_1044 = mul(x = input_161, y = var_1043)[name = tensor("op_1044")]; - tensor input_163 = add(x = var_1044, y = input_153)[name = tensor("input_163")]; - tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; - tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_163)[name = tensor("x_25")]; + tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_746)[name = tensor("x_expanded_11")]; + tensor var_760 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_760")]; + tensor var_761 = mul(x = window_15, y = var_760)[name = tensor("op_761")]; + tensor var_762 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_762")]; + tensor filled_window_11 = add(x = var_761, y = var_762)[name = tensor("filled_window_11")]; + tensor var_764 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_764")]; + tensor var_765_dtype_0 = const()[name = tensor("op_765_dtype_0"), val = tensor("fp32")]; + tensor var_766 = const()[name = tensor("op_766"), val = tensor([1, 1, 1])]; + tensor var_765 = cast(dtype = var_765_dtype_0, x = var_764)[name = tensor("cast_132")]; + tensor use_roll_11 = reshape(shape = var_766, x = var_765)[name = tensor("use_roll_11")]; + tensor var_768 = sub(x = var_17, y = use_roll_11)[name = tensor("op_768")]; + tensor var_769 = mul(x = filled_window_11, y = var_768)[name = tensor("op_769")]; + tensor var_770 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_770")]; + tensor window_17 = add(x = var_769, y = var_770)[name = tensor("window_17")]; + tensor var_772 = const()[name = tensor("op_772"), val = tensor(0x1p+0)]; + tensor var_773 = add(x = clip_9, y = var_772)[name = tensor("op_773")]; + tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_773)[name = tensor("clip_11")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_30, interleave = input_101_interleave_0, values = (window_15, window_17))[name = tensor("input_101")]; + tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; + tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; + tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_23_pad_type_0 = const()[name = tensor("inputs_23_pad_type_0"), val = tensor("valid")]; + tensor inputs_23_strides_0 = const()[name = tensor("inputs_23_strides_0"), val = tensor([1])]; + tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; + tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; + tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; + tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; + tensor var_796_split_sizes_0 = const()[name = tensor("op_796_split_sizes_0"), val = tensor([256, 256])]; + tensor var_796_axis_0 = const()[name = tensor("op_796_axis_0"), val = tensor(1)]; + tensor var_796_0, tensor var_796_1 = split(axis = var_796_axis_0, split_sizes = var_796_split_sizes_0, x = inputs_23)[name = tensor("op_796")]; + tensor var_798 = sigmoid(x = var_796_1)[name = tensor("op_798")]; + tensor inputs_25 = mul(x = var_796_0, y = var_798)[name = tensor("inputs_25")]; + tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; + tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; + tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; + tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_25)[name = tensor("outputs_aug_5")]; + tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; + tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([2, 256, 16])]; + tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; + tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_5)[name = tensor("input_105")]; + tensor inputs_27 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_105)[name = tensor("inputs_27")]; + tensor input_107 = silu(x = inputs_27)[name = tensor("input_107")]; + tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; + tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; + tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; + tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; + tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; + tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; + tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; + tensor var_829_begin_0 = const()[name = tensor("op_829_begin_0"), val = tensor([0, -1, 0])]; + tensor var_829_end_0 = const()[name = tensor("op_829_end_0"), val = tensor([2, 16, 256])]; + tensor var_829_end_mask_0 = const()[name = tensor("op_829_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_829 = slice_by_index(begin = var_829_begin_0, end = var_829_end_0, end_mask = var_829_end_mask_0, x = conv_out_5)[name = tensor("op_829")]; + tensor var_831_perm_0 = const()[name = tensor("op_831_perm_0"), val = tensor([1, 0, 2])]; + tensor var_831 = transpose(perm = var_831_perm_0, x = var_829)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_831)[name = tensor("input_111")]; + tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; + tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; + tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; + tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; + tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; + tensor var_854 = const()[name = tensor("op_854"), val = tensor(0x1p-1)]; + tensor var_855 = mul(x = input_119, y = var_854)[name = tensor("op_855")]; + tensor input_121 = add(x = var_855, y = input_111)[name = tensor("input_121")]; + tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; + tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; + tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; + tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; + tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; + tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; + tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; + tensor var_884 = const()[name = tensor("op_884"), val = tensor(0x1p-1)]; + tensor var_885 = mul(x = input_131, y = var_884)[name = tensor("op_885")]; + tensor input_133 = add(x = var_885, y = input_123)[name = tensor("input_133")]; + tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; + tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; tensor prev_kv_7_end_0 = const()[name = tensor("prev_kv_7_end_0"), val = tensor([4, 1, 4, 64, 64])]; tensor prev_kv_7_end_mask_0 = const()[name = tensor("prev_kv_7_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -984,236 +874,199 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_1058 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_25)[name = tensor("linear_30")]; - tensor var_1059 = const()[name = tensor("op_1059"), val = tensor([1, 2, 4, 64])]; - tensor var_1060 = reshape(shape = var_1059, x = var_1058)[name = tensor("op_1060")]; + tensor var_899 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_900 = const()[name = tensor("op_900"), val = tensor([1, 2, 4, 64])]; + tensor var_901 = reshape(shape = var_900, x = var_899)[name = tensor("op_901")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1064 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_25)[name = tensor("linear_31")]; - tensor var_1065 = const()[name = tensor("op_1065"), val = tensor(0x1p-3)]; - tensor var_1066 = mul(x = var_1064, y = var_1065)[name = tensor("op_1066")]; - tensor var_1067 = const()[name = tensor("op_1067"), val = tensor([1, 2, 4, 64])]; - tensor var_1068 = reshape(shape = var_1067, x = var_1066)[name = tensor("op_1068")]; + tensor var_905 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_906 = const()[name = tensor("op_906"), val = tensor(0x1p-3)]; + tensor var_907 = mul(x = var_905, y = var_906)[name = tensor("op_907")]; + tensor var_908 = const()[name = tensor("op_908"), val = tensor([1, 2, 4, 64])]; + tensor var_909 = reshape(shape = var_908, x = var_907)[name = tensor("op_909")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1072 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_25)[name = tensor("linear_32")]; - tensor var_1073 = const()[name = tensor("op_1073"), val = tensor([1, 2, 4, 64])]; - tensor var_1074 = reshape(shape = var_1073, x = var_1072)[name = tensor("op_1074")]; + tensor var_913 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_914 = const()[name = tensor("op_914"), val = tensor([1, 2, 4, 64])]; + tensor var_915 = reshape(shape = var_914, x = var_913)[name = tensor("op_915")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_167 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_25)[name = tensor("linear_33")]; + tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; tensor s_t_7 = add(x = prev_scale_7, y = encoder__t_index)[name = tensor("s_t_7")]; tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_1068)[name = tensor("transpose_37")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_1060)[name = tensor("transpose_38")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_909)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_901)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_1084 = const()[name = tensor("op_1084"), val = tensor([2, 1])]; - tensor var_1085 = reshape(shape = var_1084, x = sqrt_s_t_7)[name = tensor("op_1085")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_1085)[name = tensor("M_7")]; - tensor var_1087 = mul(x = qk_7, y = M_7)[name = tensor("op_1087")]; + tensor var_925 = const()[name = tensor("op_925"), val = tensor([2, 1])]; + tensor var_926 = reshape(shape = var_925, x = sqrt_s_t_7)[name = tensor("op_926")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_926)[name = tensor("M_7")]; + tensor var_928 = mul(x = qk_7, y = M_7)[name = tensor("op_928")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_1074)[name = tensor("transpose_36")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1087, y = v_7)[name = tensor("inner_7")]; - tensor var_1089_transpose_x_0 = const()[name = tensor("op_1089_transpose_x_0"), val = tensor(false)]; - tensor var_1089_transpose_y_0 = const()[name = tensor("op_1089_transpose_y_0"), val = tensor(false)]; - tensor var_1089 = matmul(transpose_x = var_1089_transpose_x_0, transpose_y = var_1089_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1089")]; - tensor var_1090 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1090")]; - tensor var_1091 = const()[name = tensor("op_1091"), val = tensor([1, 1, 2, 1])]; - tensor var_1092 = reshape(shape = var_1091, x = var_1090)[name = tensor("op_1092")]; - tensor cross_7 = mul(x = var_1089, y = var_1092)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_915)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_928, y = v_7)[name = tensor("inner_7")]; + tensor var_930_transpose_x_0 = const()[name = tensor("op_930_transpose_x_0"), val = tensor(false)]; + tensor var_930_transpose_y_0 = const()[name = tensor("op_930_transpose_y_0"), val = tensor(false)]; + tensor var_930 = matmul(transpose_x = var_930_transpose_x_0, transpose_y = var_930_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_930")]; + tensor var_931 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_931")]; + tensor var_932 = const()[name = tensor("op_932"), val = tensor([1, 1, 2, 1])]; + tensor var_933 = reshape(shape = var_932, x = var_931)[name = tensor("op_933")]; + tensor cross_7 = mul(x = var_930, y = var_933)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_1095 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1095")]; - tensor var_1097_transpose_x_1 = const()[name = tensor("op_1097_transpose_x_1"), val = tensor(true)]; - tensor var_1097_transpose_y_1 = const()[name = tensor("op_1097_transpose_y_1"), val = tensor(false)]; - tensor var_1097 = matmul(transpose_x = var_1097_transpose_x_1, transpose_y = var_1097_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1097")]; - tensor new_kv_unnorm_7 = add(x = var_1095, y = var_1097)[name = tensor("new_kv_unnorm_7")]; - tensor var_1099 = const()[name = tensor("op_1099"), val = tensor(0x1p+1)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_1099)[name = tensor("new_scale_7")]; - tensor var_1101 = sqrt(x = new_scale_7)[name = tensor("op_1101")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1101)[name = tensor("nkv_1")]; - tensor var_1103_perm_0 = const()[name = tensor("op_1103_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_936 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_936")]; + tensor var_938_transpose_x_1 = const()[name = tensor("op_938_transpose_x_1"), val = tensor(true)]; + tensor var_938_transpose_y_1 = const()[name = tensor("op_938_transpose_y_1"), val = tensor(false)]; + tensor var_938 = matmul(transpose_x = var_938_transpose_x_1, transpose_y = var_938_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_938")]; + tensor new_kv_unnorm_7 = add(x = var_936, y = var_938)[name = tensor("new_kv_unnorm_7")]; + tensor var_940 = const()[name = tensor("op_940"), val = tensor(0x1p+1)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_940)[name = tensor("new_scale_7")]; + tensor var_942 = sqrt(x = new_scale_7)[name = tensor("op_942")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_942)[name = tensor("nkv_1")]; + tensor var_944_perm_0 = const()[name = tensor("op_944_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_1103 = transpose(perm = var_1103_perm_0, x = out_19)[name = tensor("transpose_35")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1103)[name = tensor("out_21")]; - tensor var_1107 = const()[name = tensor("op_1107"), val = tensor([1, 2, 256])]; - tensor out_23 = reshape(shape = var_1107, x = out_21)[name = tensor("out_23")]; - tensor var_1109 = silu(x = input_167)[name = tensor("op_1109")]; - tensor input_169 = mul(x = var_1109, y = out_23)[name = tensor("input_169")]; - tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_169)[name = tensor("linear_34")]; - tensor x_27 = add(x = input_163, y = ret_out_7)[name = tensor("x_27")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor var_944 = transpose(perm = var_944_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_944)[name = tensor("out_21")]; + tensor var_948 = const()[name = tensor("op_948"), val = tensor([1, 2, 256])]; + tensor out_23 = reshape(shape = var_948, x = out_21)[name = tensor("out_23")]; + tensor var_950 = silu(x = input_137)[name = tensor("op_950")]; + tensor input_139 = mul(x = var_950, y = out_23)[name = tensor("input_139")]; + tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; + tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; + tensor window_19_begin_0 = const()[name = tensor("window_19_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_19_end_0 = const()[name = tensor("window_19_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_19_end_mask_0 = const()[name = tensor("window_19_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_19_squeeze_mask_0 = const()[name = tensor("window_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_19 = slice_by_index(begin = window_19_begin_0, end = window_19_end_0, end_mask = window_19_end_mask_0, squeeze_mask = window_19_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_19")]; tensor filled_13_begin_0 = const()[name = tensor("filled_13_begin_0"), val = tensor([3, 0])]; tensor filled_13_end_0 = const()[name = tensor("filled_13_end_0"), val = tensor([4, 1])]; tensor filled_13_end_mask_0 = const()[name = tensor("filled_13_end_mask_0"), val = tensor([false, true])]; tensor filled_13_squeeze_mask_0 = const()[name = tensor("filled_13_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_13 = slice_by_index(begin = filled_13_begin_0, end = filled_13_end_0, end_mask = filled_13_end_mask_0, squeeze_mask = filled_13_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_13")]; - tensor var_1118_begin_0 = const()[name = tensor("op_1118_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1118_end_0 = const()[name = tensor("op_1118_end_0"), val = tensor([1, 1, 256])]; - tensor var_1118_end_mask_0 = const()[name = tensor("op_1118_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1118 = slice_by_index(begin = var_1118_begin_0, end = var_1118_end_0, end_mask = var_1118_end_mask_0, x = x_27)[name = tensor("op_1118")]; - tensor var_1121_begin_0 = const()[name = tensor("op_1121_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1121_end_0 = const()[name = tensor("op_1121_end_0"), val = tensor([1, 16, 256])]; - tensor var_1121_end_mask_0 = const()[name = tensor("op_1121_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1121 = slice_by_index(begin = var_1121_begin_0, end = var_1121_end_0, end_mask = var_1121_end_mask_0, x = window)[name = tensor("op_1121")]; + tensor var_959_begin_0 = const()[name = tensor("op_959_begin_0"), val = tensor([0, 0, 0])]; + tensor var_959_end_0 = const()[name = tensor("op_959_end_0"), val = tensor([1, 1, 256])]; + tensor var_959_end_mask_0 = const()[name = tensor("op_959_end_mask_0"), val = tensor([true, false, true])]; + tensor var_959 = slice_by_index(begin = var_959_begin_0, end = var_959_end_0, end_mask = var_959_end_mask_0, x = x_21)[name = tensor("op_959")]; + tensor var_962_begin_0 = const()[name = tensor("op_962_begin_0"), val = tensor([0, 1, 0])]; + tensor var_962_end_0 = const()[name = tensor("op_962_end_0"), val = tensor([1, 16, 256])]; + tensor var_962_end_mask_0 = const()[name = tensor("op_962_end_mask_0"), val = tensor([true, true, true])]; + tensor var_962 = slice_by_index(begin = var_962_begin_0, end = var_962_end_0, end_mask = var_962_end_mask_0, x = window_19)[name = tensor("op_962")]; tensor rolled_13_interleave_0 = const()[name = tensor("rolled_13_interleave_0"), val = tensor(false)]; - tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_1121, var_1118))[name = tensor("rolled_13")]; + tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_962, var_959))[name = tensor("rolled_13")]; tensor clip_12 = clip(alpha = var_21, beta = var_20, x = filled_13)[name = tensor("clip_12")]; - tensor var_1126 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_1126")]; - tensor var_1127_dtype_0 = const()[name = tensor("op_1127_dtype_0"), val = tensor("fp32")]; - tensor var_1128 = const()[name = tensor("op_1128"), val = tensor([1, 16, 1])]; - tensor var_1127 = cast(dtype = var_1127_dtype_0, x = var_1126)[name = tensor("cast_131")]; - tensor fill_mask_13 = reshape(shape = var_1128, x = var_1127)[name = tensor("fill_mask_13")]; + tensor var_967 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_967")]; + tensor var_968_dtype_0 = const()[name = tensor("op_968_dtype_0"), val = tensor("fp32")]; + tensor var_969 = const()[name = tensor("op_969"), val = tensor([1, 16, 1])]; + tensor var_968 = cast(dtype = var_968_dtype_0, x = var_967)[name = tensor("cast_131")]; + tensor fill_mask_13 = reshape(shape = var_969, x = var_968)[name = tensor("fill_mask_13")]; tensor x_expanded_13_reps_0 = const()[name = tensor("x_expanded_13_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_1118)[name = tensor("x_expanded_13")]; - tensor var_1132 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_1132")]; - tensor var_1133 = mul(x = window, y = var_1132)[name = tensor("op_1133")]; - tensor var_1134 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_1134")]; - tensor filled_window_13 = add(x = var_1133, y = var_1134)[name = tensor("filled_window_13")]; - tensor var_1136 = greater_equal(x = filled_13, y = var_16)[name = tensor("op_1136")]; - tensor var_1137_dtype_0 = const()[name = tensor("op_1137_dtype_0"), val = tensor("fp32")]; - tensor var_1138 = const()[name = tensor("op_1138"), val = tensor([1, 1, 1])]; - tensor var_1137 = cast(dtype = var_1137_dtype_0, x = var_1136)[name = tensor("cast_130")]; - tensor use_roll_13 = reshape(shape = var_1138, x = var_1137)[name = tensor("use_roll_13")]; - tensor var_1140 = sub(x = var_17, y = use_roll_13)[name = tensor("op_1140")]; - tensor var_1141 = mul(x = filled_window_13, y = var_1140)[name = tensor("op_1141")]; - tensor var_1142 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_1142")]; - tensor input_171 = add(x = var_1141, y = var_1142)[name = tensor("input_171")]; - tensor var_1144 = const()[name = tensor("op_1144"), val = tensor(0x1p+0)]; - tensor var_1145 = add(x = filled_13, y = var_1144)[name = tensor("op_1145")]; - tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_1145)[name = tensor("clip_13")]; - tensor x_29_axes_0 = const()[name = tensor("x_29_axes_0"), val = tensor([-1])]; - tensor x_29 = layer_norm(axes = x_29_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_171)[name = tensor("x_29")]; - tensor input_173_perm_0 = const()[name = tensor("input_173_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_51_pad_type_0 = const()[name = tensor("inputs_51_pad_type_0"), val = tensor("valid")]; - tensor inputs_51_strides_0 = const()[name = tensor("inputs_51_strides_0"), val = tensor([1])]; - tensor inputs_51_pad_0 = const()[name = tensor("inputs_51_pad_0"), val = tensor([0, 0])]; - tensor inputs_51_dilations_0 = const()[name = tensor("inputs_51_dilations_0"), val = tensor([1])]; - tensor inputs_51_groups_0 = const()[name = tensor("inputs_51_groups_0"), val = tensor(1)]; - tensor input_173 = transpose(perm = input_173_perm_0, x = x_29)[name = tensor("transpose_34")]; - tensor inputs_51 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_51_dilations_0, groups = inputs_51_groups_0, pad = inputs_51_pad_0, pad_type = inputs_51_pad_type_0, strides = inputs_51_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_173)[name = tensor("inputs_51")]; - tensor var_1166_split_sizes_0 = const()[name = tensor("op_1166_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1166_axis_0 = const()[name = tensor("op_1166_axis_0"), val = tensor(1)]; - tensor var_1166_0, tensor var_1166_1 = split(axis = var_1166_axis_0, split_sizes = var_1166_split_sizes_0, x = inputs_51)[name = tensor("op_1166")]; - tensor var_1168 = sigmoid(x = var_1166_1)[name = tensor("op_1168")]; - tensor inputs_53 = mul(x = var_1166_0, y = var_1168)[name = tensor("inputs_53")]; - tensor outputs_aug_13_pad_type_0 = const()[name = tensor("outputs_aug_13_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_13_pad_0 = const()[name = tensor("outputs_aug_13_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_13_groups_0 = const()[name = tensor("outputs_aug_13_groups_0"), val = tensor(256)]; - tensor outputs_aug_13_strides_0 = const()[name = tensor("outputs_aug_13_strides_0"), val = tensor([1])]; - tensor outputs_aug_13_dilations_0 = const()[name = tensor("outputs_aug_13_dilations_0"), val = tensor([1])]; - tensor outputs_aug_13 = conv(dilations = outputs_aug_13_dilations_0, groups = outputs_aug_13_groups_0, pad = outputs_aug_13_pad_0, pad_type = outputs_aug_13_pad_type_0, strides = outputs_aug_13_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_53)[name = tensor("outputs_aug_13")]; - tensor input_175_begin_0 = const()[name = tensor("input_175_begin_0"), val = tensor([0, 0, 0])]; - tensor input_175_end_0 = const()[name = tensor("input_175_end_0"), val = tensor([1, 256, 16])]; - tensor input_175_end_mask_0 = const()[name = tensor("input_175_end_mask_0"), val = tensor([true, true, false])]; - tensor input_175 = slice_by_index(begin = input_175_begin_0, end = input_175_end_0, end_mask = input_175_end_mask_0, x = outputs_aug_13)[name = tensor("input_175")]; - tensor inputs_55 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_175)[name = tensor("inputs_55")]; - tensor input_177 = silu(x = inputs_55)[name = tensor("input_177")]; - tensor input_179_pad_type_0 = const()[name = tensor("input_179_pad_type_0"), val = tensor("valid")]; - tensor input_179_strides_0 = const()[name = tensor("input_179_strides_0"), val = tensor([1])]; - tensor input_179_pad_0 = const()[name = tensor("input_179_pad_0"), val = tensor([0, 0])]; - tensor input_179_dilations_0 = const()[name = tensor("input_179_dilations_0"), val = tensor([1])]; - tensor input_179_groups_0 = const()[name = tensor("input_179_groups_0"), val = tensor(1)]; - tensor input_179 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_179_dilations_0, groups = input_179_groups_0, pad = input_179_pad_0, pad_type = input_179_pad_type_0, strides = input_179_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_177)[name = tensor("input_179")]; - tensor conv_out_13_perm_0 = const()[name = tensor("conv_out_13_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1199_begin_0 = const()[name = tensor("op_1199_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1199_end_0 = const()[name = tensor("op_1199_end_0"), val = tensor([1, 16, 256])]; - tensor var_1199_end_mask_0 = const()[name = tensor("op_1199_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_13 = transpose(perm = conv_out_13_perm_0, x = input_179)[name = tensor("transpose_33")]; - tensor var_1199 = slice_by_index(begin = var_1199_begin_0, end = var_1199_end_0, end_mask = var_1199_end_mask_0, x = conv_out_13)[name = tensor("op_1199")]; - tensor var_1202_begin_0 = const()[name = tensor("op_1202_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1202_end_0 = const()[name = tensor("op_1202_end_0"), val = tensor([1, 1, 256])]; - tensor var_1202_end_mask_0 = const()[name = tensor("op_1202_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1202 = slice_by_index(begin = var_1202_begin_0, end = var_1202_end_0, end_mask = var_1202_end_mask_0, x = x_27)[name = tensor("op_1202")]; - tensor var_1205_begin_0 = const()[name = tensor("op_1205_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1205_end_0 = const()[name = tensor("op_1205_end_0"), val = tensor([1, 16, 256])]; - tensor var_1205_end_mask_0 = const()[name = tensor("op_1205_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1205 = slice_by_index(begin = var_1205_begin_0, end = var_1205_end_0, end_mask = var_1205_end_mask_0, x = input_171)[name = tensor("op_1205")]; + tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_959)[name = tensor("x_expanded_13")]; + tensor var_973 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_973")]; + tensor var_974 = mul(x = window_19, y = var_973)[name = tensor("op_974")]; + tensor var_975 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_975")]; + tensor filled_window_13 = add(x = var_974, y = var_975)[name = tensor("filled_window_13")]; + tensor var_977 = greater_equal(x = filled_13, y = var_16)[name = tensor("op_977")]; + tensor var_978_dtype_0 = const()[name = tensor("op_978_dtype_0"), val = tensor("fp32")]; + tensor var_979 = const()[name = tensor("op_979"), val = tensor([1, 1, 1])]; + tensor var_978 = cast(dtype = var_978_dtype_0, x = var_977)[name = tensor("cast_130")]; + tensor use_roll_13 = reshape(shape = var_979, x = var_978)[name = tensor("use_roll_13")]; + tensor var_981 = sub(x = var_17, y = use_roll_13)[name = tensor("op_981")]; + tensor var_982 = mul(x = filled_window_13, y = var_981)[name = tensor("op_982")]; + tensor var_983 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_983")]; + tensor window_21 = add(x = var_982, y = var_983)[name = tensor("window_21")]; + tensor var_985 = const()[name = tensor("op_985"), val = tensor(0x1p+0)]; + tensor var_986 = add(x = filled_13, y = var_985)[name = tensor("op_986")]; + tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_986)[name = tensor("clip_13")]; + tensor var_989_begin_0 = const()[name = tensor("op_989_begin_0"), val = tensor([0, 1, 0])]; + tensor var_989_end_0 = const()[name = tensor("op_989_end_0"), val = tensor([1, 1, 256])]; + tensor var_989_end_mask_0 = const()[name = tensor("op_989_end_mask_0"), val = tensor([true, true, true])]; + tensor var_989 = slice_by_index(begin = var_989_begin_0, end = var_989_end_0, end_mask = var_989_end_mask_0, x = x_21)[name = tensor("op_989")]; + tensor var_992_begin_0 = const()[name = tensor("op_992_begin_0"), val = tensor([0, 1, 0])]; + tensor var_992_end_0 = const()[name = tensor("op_992_end_0"), val = tensor([1, 16, 256])]; + tensor var_992_end_mask_0 = const()[name = tensor("op_992_end_mask_0"), val = tensor([true, true, true])]; + tensor var_992 = slice_by_index(begin = var_992_begin_0, end = var_992_end_0, end_mask = var_992_end_mask_0, x = window_21)[name = tensor("op_992")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1205, var_1202))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_992, var_989))[name = tensor("rolled")]; tensor clip_14 = clip(alpha = var_21, beta = var_20, x = clip_13)[name = tensor("clip_14")]; - tensor var_1210 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_1210")]; - tensor var_1211_dtype_0 = const()[name = tensor("op_1211_dtype_0"), val = tensor("fp32")]; - tensor var_1212 = const()[name = tensor("op_1212"), val = tensor([1, 16, 1])]; - tensor var_1211 = cast(dtype = var_1211_dtype_0, x = var_1210)[name = tensor("cast_129")]; - tensor fill_mask = reshape(shape = var_1212, x = var_1211)[name = tensor("fill_mask")]; + tensor var_997 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_997")]; + tensor var_998_dtype_0 = const()[name = tensor("op_998_dtype_0"), val = tensor("fp32")]; + tensor var_999 = const()[name = tensor("op_999"), val = tensor([1, 16, 1])]; + tensor var_998 = cast(dtype = var_998_dtype_0, x = var_997)[name = tensor("cast_129")]; + tensor fill_mask = reshape(shape = var_999, x = var_998)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1202)[name = tensor("x_expanded")]; - tensor var_1216 = sub(x = var_17, y = fill_mask)[name = tensor("op_1216")]; - tensor var_1217 = mul(x = input_171, y = var_1216)[name = tensor("op_1217")]; - tensor var_1218 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1218")]; - tensor filled_window = add(x = var_1217, y = var_1218)[name = tensor("filled_window")]; - tensor var_1220 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_1220")]; - tensor var_1221_dtype_0 = const()[name = tensor("op_1221_dtype_0"), val = tensor("fp32")]; - tensor var_1222 = const()[name = tensor("op_1222"), val = tensor([1, 1, 1])]; - tensor var_1221 = cast(dtype = var_1221_dtype_0, x = var_1220)[name = tensor("cast_128")]; - tensor use_roll = reshape(shape = var_1222, x = var_1221)[name = tensor("use_roll")]; - tensor var_1224 = sub(x = var_17, y = use_roll)[name = tensor("op_1224")]; - tensor var_1225 = mul(x = filled_window, y = var_1224)[name = tensor("op_1225")]; - tensor var_1226 = mul(x = rolled, y = use_roll)[name = tensor("op_1226")]; - tensor input_181 = add(x = var_1225, y = var_1226)[name = tensor("input_181")]; - tensor var_1228 = const()[name = tensor("op_1228"), val = tensor(0x1p+0)]; - tensor var_1229 = add(x = clip_13, y = var_1228)[name = tensor("op_1229")]; - tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_1229)[name = tensor("clip_15")]; - tensor x_31_axes_0 = const()[name = tensor("x_31_axes_0"), val = tensor([-1])]; - tensor x_31 = layer_norm(axes = x_31_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_181)[name = tensor("x_31")]; - tensor input_183_perm_0 = const()[name = tensor("input_183_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_57_pad_type_0 = const()[name = tensor("inputs_57_pad_type_0"), val = tensor("valid")]; - tensor inputs_57_strides_0 = const()[name = tensor("inputs_57_strides_0"), val = tensor([1])]; - tensor inputs_57_pad_0 = const()[name = tensor("inputs_57_pad_0"), val = tensor([0, 0])]; - tensor inputs_57_dilations_0 = const()[name = tensor("inputs_57_dilations_0"), val = tensor([1])]; - tensor inputs_57_groups_0 = const()[name = tensor("inputs_57_groups_0"), val = tensor(1)]; - tensor input_183 = transpose(perm = input_183_perm_0, x = x_31)[name = tensor("transpose_32")]; - tensor inputs_57 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_57_dilations_0, groups = inputs_57_groups_0, pad = inputs_57_pad_0, pad_type = inputs_57_pad_type_0, strides = inputs_57_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_183)[name = tensor("inputs_57")]; - tensor var_1250_split_sizes_0 = const()[name = tensor("op_1250_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1250_axis_0 = const()[name = tensor("op_1250_axis_0"), val = tensor(1)]; - tensor var_1250_0, tensor var_1250_1 = split(axis = var_1250_axis_0, split_sizes = var_1250_split_sizes_0, x = inputs_57)[name = tensor("op_1250")]; - tensor var_1252 = sigmoid(x = var_1250_1)[name = tensor("op_1252")]; - tensor inputs_59 = mul(x = var_1250_0, y = var_1252)[name = tensor("inputs_59")]; + tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_989)[name = tensor("x_expanded")]; + tensor var_1003 = sub(x = var_17, y = fill_mask)[name = tensor("op_1003")]; + tensor var_1004 = mul(x = window_21, y = var_1003)[name = tensor("op_1004")]; + tensor var_1005 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1005")]; + tensor filled_window = add(x = var_1004, y = var_1005)[name = tensor("filled_window")]; + tensor var_1007 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_1007")]; + tensor var_1008_dtype_0 = const()[name = tensor("op_1008_dtype_0"), val = tensor("fp32")]; + tensor var_1009 = const()[name = tensor("op_1009"), val = tensor([1, 1, 1])]; + tensor var_1008 = cast(dtype = var_1008_dtype_0, x = var_1007)[name = tensor("cast_128")]; + tensor use_roll = reshape(shape = var_1009, x = var_1008)[name = tensor("use_roll")]; + tensor var_1011 = sub(x = var_17, y = use_roll)[name = tensor("op_1011")]; + tensor var_1012 = mul(x = filled_window, y = var_1011)[name = tensor("op_1012")]; + tensor var_1013 = mul(x = rolled, y = use_roll)[name = tensor("op_1013")]; + tensor window = add(x = var_1012, y = var_1013)[name = tensor("window")]; + tensor var_1015 = const()[name = tensor("op_1015"), val = tensor(0x1p+0)]; + tensor var_1016 = add(x = clip_13, y = var_1015)[name = tensor("op_1016")]; + tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_1016)[name = tensor("clip_15")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_30, interleave = input_141_interleave_0, values = (window_21, window))[name = tensor("input_141")]; + tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; + tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; + tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_33_pad_type_0 = const()[name = tensor("inputs_33_pad_type_0"), val = tensor("valid")]; + tensor inputs_33_strides_0 = const()[name = tensor("inputs_33_strides_0"), val = tensor([1])]; + tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; + tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; + tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; + tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; + tensor var_1039_split_sizes_0 = const()[name = tensor("op_1039_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1039_axis_0 = const()[name = tensor("op_1039_axis_0"), val = tensor(1)]; + tensor var_1039_0, tensor var_1039_1 = split(axis = var_1039_axis_0, split_sizes = var_1039_split_sizes_0, x = inputs_33)[name = tensor("op_1039")]; + tensor var_1041 = sigmoid(x = var_1039_1)[name = tensor("op_1041")]; + tensor inputs_35 = mul(x = var_1039_0, y = var_1041)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; tensor outputs_aug_strides_0 = const()[name = tensor("outputs_aug_strides_0"), val = tensor([1])]; tensor outputs_aug_dilations_0 = const()[name = tensor("outputs_aug_dilations_0"), val = tensor([1])]; - tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_59)[name = tensor("outputs_aug")]; - tensor input_185_begin_0 = const()[name = tensor("input_185_begin_0"), val = tensor([0, 0, 0])]; - tensor input_185_end_0 = const()[name = tensor("input_185_end_0"), val = tensor([1, 256, 16])]; - tensor input_185_end_mask_0 = const()[name = tensor("input_185_end_mask_0"), val = tensor([true, true, false])]; - tensor input_185 = slice_by_index(begin = input_185_begin_0, end = input_185_end_0, end_mask = input_185_end_mask_0, x = outputs_aug)[name = tensor("input_185")]; - tensor inputs_61 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_185)[name = tensor("inputs_61")]; - tensor input_187 = silu(x = inputs_61)[name = tensor("input_187")]; - tensor input_189_pad_type_0 = const()[name = tensor("input_189_pad_type_0"), val = tensor("valid")]; - tensor input_189_strides_0 = const()[name = tensor("input_189_strides_0"), val = tensor([1])]; - tensor input_189_pad_0 = const()[name = tensor("input_189_pad_0"), val = tensor([0, 0])]; - tensor input_189_dilations_0 = const()[name = tensor("input_189_dilations_0"), val = tensor([1])]; - tensor input_189_groups_0 = const()[name = tensor("input_189_groups_0"), val = tensor(1)]; - tensor input_189 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_189_dilations_0, groups = input_189_groups_0, pad = input_189_pad_0, pad_type = input_189_pad_type_0, strides = input_189_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_187)[name = tensor("input_189")]; - tensor conv_out_15_perm_0 = const()[name = tensor("conv_out_15_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1283_begin_0 = const()[name = tensor("op_1283_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1283_end_0 = const()[name = tensor("op_1283_end_0"), val = tensor([1, 16, 256])]; - tensor var_1283_end_mask_0 = const()[name = tensor("op_1283_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_15 = transpose(perm = conv_out_15_perm_0, x = input_189)[name = tensor("transpose_31")]; - tensor var_1283 = slice_by_index(begin = var_1283_begin_0, end = var_1283_end_0, end_mask = var_1283_end_mask_0, x = conv_out_15)[name = tensor("op_1283")]; - tensor var_1286_interleave_0 = const()[name = tensor("op_1286_interleave_0"), val = tensor(false)]; - tensor var_1286 = concat(axis = var_33, interleave = var_1286_interleave_0, values = (var_1199, var_1283))[name = tensor("op_1286")]; - tensor input_191 = add(x = x_27, y = var_1286)[name = tensor("input_191")]; - tensor input_193_axes_0 = const()[name = tensor("input_193_axes_0"), val = tensor([-1])]; - tensor input_193 = layer_norm(axes = input_193_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_191)[name = tensor("input_193")]; - tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_193)[name = tensor("linear_35")]; - tensor input_195 = silu(x = inputs)[name = tensor("input_195")]; - tensor input_199 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_195)[name = tensor("linear_36")]; - tensor var_1309 = const()[name = tensor("op_1309"), val = tensor(0x1p-1)]; - tensor var_1310 = mul(x = input_199, y = var_1309)[name = tensor("op_1310")]; - tensor input_201 = add(x = var_1310, y = input_191)[name = tensor("input_201")]; - tensor x_33_axes_0 = const()[name = tensor("x_33_axes_0"), val = tensor([-1])]; - tensor x_33 = layer_norm(axes = x_33_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_201)[name = tensor("x_33")]; + tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_35)[name = tensor("outputs_aug")]; + tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; + tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([2, 256, 16])]; + tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; + tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug)[name = tensor("input_145")]; + tensor inputs_37 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_145)[name = tensor("inputs_37")]; + tensor input_147 = silu(x = inputs_37)[name = tensor("input_147")]; + tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; + tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; + tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; + tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; + tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; + tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; + tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1072_begin_0 = const()[name = tensor("op_1072_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1072_end_0 = const()[name = tensor("op_1072_end_0"), val = tensor([2, 16, 256])]; + tensor var_1072_end_mask_0 = const()[name = tensor("op_1072_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_1072 = slice_by_index(begin = var_1072_begin_0, end = var_1072_end_0, end_mask = var_1072_end_mask_0, x = conv_out_7)[name = tensor("op_1072")]; + tensor var_1074_perm_0 = const()[name = tensor("op_1074_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1074 = transpose(perm = var_1074_perm_0, x = var_1072)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_1074)[name = tensor("input_151")]; + tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; + tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; + tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; + tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; + tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; + tensor var_1097 = const()[name = tensor("op_1097"), val = tensor(0x1p-1)]; + tensor var_1098 = mul(x = input_159, y = var_1097)[name = tensor("op_1098")]; + tensor input_161 = add(x = var_1098, y = input_151)[name = tensor("input_161")]; + tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; + tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; tensor cat_interleave_0 = const()[name = tensor("cat_interleave_0"), val = tensor(false)]; - tensor x_ct = transpose(perm = x_ct_perm_0, x = x_33)[name = tensor("transpose_30")]; + tensor x_ct = transpose(perm = x_ct_perm_0, x = x_25)[name = tensor("transpose_30")]; tensor cat = concat(axis = var_27, interleave = cat_interleave_0, values = (cnn_window, x_ct))[name = tensor("cat")]; tensor conv_out_pad_type_0 = const()[name = tensor("conv_out_pad_type_0"), val = tensor("valid")]; tensor conv_out_strides_0 = const()[name = tensor("conv_out_strides_0"), val = tensor([1])]; @@ -1221,44 +1074,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_1328_begin_0 = const()[name = tensor("op_1328_begin_0"), val = tensor([0, 0, 2])]; - tensor var_1328_end_0 = const()[name = tensor("op_1328_end_0"), val = tensor([1, 256, 20])]; - tensor var_1328_end_mask_0 = const()[name = tensor("op_1328_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_1328_begin_0, end = var_1328_end_0, end_mask = var_1328_end_mask_0, x = cat)[name = tensor("op_1328")]; - tensor input_203_perm_0 = const()[name = tensor("input_203_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1330 = const()[name = tensor("op_1330"), val = tensor([-1])]; - tensor input_203 = transpose(perm = input_203_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_1331 = reduce_l2_norm(axes = var_1330, keep_dims = var_36, x = input_203)[name = tensor("op_1331")]; - tensor const_16 = const()[name = tensor("const_16"), val = tensor(0x1.fffffep+127)]; - tensor clip_16 = clip(alpha = var_12, beta = const_16, x = var_1331)[name = tensor("clip_16")]; - tensor emb = real_div(x = input_203, y = clip_16)[name = tensor("emb")]; - tensor var_1335_axis_0 = const()[name = tensor("op_1335_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_1335_axis_0, values = (var_214, var_510, var_806, nkv_1))[name = tensor("op_1335")]; - tensor var_1337_axis_0 = const()[name = tensor("op_1337_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_1337_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1337")]; - tensor var_1339_axis_0 = const()[name = tensor("op_1339_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_1339_axis_0, values = (input_31, input_81, input_131, input_181))[name = tensor("op_1339")]; - tensor var_1341_axis_0 = const()[name = tensor("op_1341_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_1341_axis_0, values = (clip_3, clip_7, clip_11, clip_15))[name = tensor("op_1341")]; - tensor var_1351 = const()[name = tensor("op_1351"), val = tensor(0x1.5798eep-27)]; - tensor var_1357 = const()[name = tensor("op_1357"), val = tensor(0x1.4f8b58p-17)]; - tensor var_1359 = const()[name = tensor("op_1359"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_1360 = const()[name = tensor("op_1360"), val = tensor(true)]; - tensor var_1362 = const()[name = tensor("op_1362"), val = tensor(0x1p+0)]; - tensor var_1366 = const()[name = tensor("op_1366"), val = tensor(-1)]; - tensor var_1371 = const()[name = tensor("op_1371"), val = tensor(0)]; + tensor var_1116_begin_0 = const()[name = tensor("op_1116_begin_0"), val = tensor([0, 0, 2])]; + tensor var_1116_end_0 = const()[name = tensor("op_1116_end_0"), val = tensor([1, 256, 20])]; + tensor var_1116_end_mask_0 = const()[name = tensor("op_1116_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_1116_begin_0, end = var_1116_end_0, end_mask = var_1116_end_mask_0, x = cat)[name = tensor("op_1116")]; + tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1118 = const()[name = tensor("op_1118"), val = tensor([-1])]; + tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; + tensor var_1119 = reduce_l2_norm(axes = var_1118, keep_dims = var_36, x = input_163)[name = tensor("op_1119")]; + tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; + tensor clip_16 = clip(alpha = var_12, beta = const_12, x = var_1119)[name = tensor("clip_16")]; + tensor emb = real_div(x = input_163, y = clip_16)[name = tensor("emb")]; + tensor var_1123_axis_0 = const()[name = tensor("op_1123_axis_0"), val = tensor(0)]; + tensor enc_kv_new = stack(axis = var_1123_axis_0, values = (var_214, var_457, var_700, nkv_1))[name = tensor("op_1123")]; + tensor var_1125_axis_0 = const()[name = tensor("op_1125_axis_0"), val = tensor(0)]; + tensor enc_scale_new = stack(axis = var_1125_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1125")]; + tensor var_1127_axis_0 = const()[name = tensor("op_1127_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1127_axis_0, values = (window_5, window_11, window_17, window))[name = tensor("op_1127")]; + tensor var_1129_axis_0 = const()[name = tensor("op_1129_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1129_axis_0, values = (clip_3, clip_7, clip_11, clip_15))[name = tensor("op_1129")]; + tensor var_1139 = const()[name = tensor("op_1139"), val = tensor(0x1.5798eep-27)]; + tensor var_1144 = const()[name = tensor("op_1144"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1146 = const()[name = tensor("op_1146"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1147 = const()[name = tensor("op_1147"), val = tensor(true)]; + tensor var_1149 = const()[name = tensor("op_1149"), val = tensor(0x1p+0)]; + tensor var_1153 = const()[name = tensor("op_1153"), val = tensor(-1)]; + tensor var_1159 = const()[name = tensor("op_1159"), val = tensor(0)]; tensor pos = const()[name = tensor("pos"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395456)))]; - tensor var_1433_axes_0 = const()[name = tensor("op_1433_axes_0"), val = tensor([2])]; - tensor var_1433 = expand_dims(axes = var_1433_axes_0, x = emb)[name = tensor("op_1433")]; - tensor var_1434 = const()[name = tensor("op_1434"), val = tensor([1, 1, 6, 1])]; - tensor emb_exp = tile(reps = var_1434, x = var_1433)[name = tensor("emb_exp")]; - tensor input_205_interleave_0 = const()[name = tensor("input_205_interleave_0"), val = tensor(false)]; - tensor input_205 = concat(axis = var_1366, interleave = input_205_interleave_0, values = (emb_exp, pos))[name = tensor("input_205")]; - tensor x_35 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_205)[name = tensor("linear_37")]; - tensor var_1441_perm_0 = const()[name = tensor("op_1441_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1445 = const()[name = tensor("op_1445"), val = tensor([6, 2, 256])]; - tensor var_1441 = transpose(perm = var_1441_perm_0, x = x_35)[name = tensor("transpose_28")]; - tensor x_37 = reshape(shape = var_1445, x = var_1441)[name = tensor("x_37")]; + tensor var_1221_axes_0 = const()[name = tensor("op_1221_axes_0"), val = tensor([2])]; + tensor var_1221 = expand_dims(axes = var_1221_axes_0, x = emb)[name = tensor("op_1221")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 6, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1221)[name = tensor("emb_exp")]; + tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; + tensor input_165 = concat(axis = var_1153, interleave = input_165_interleave_0, values = (emb_exp, pos))[name = tensor("input_165")]; + tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; + tensor var_1229_perm_0 = const()[name = tensor("op_1229_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1233 = const()[name = tensor("op_1233"), val = tensor([6, 2, 256])]; + tensor var_1229 = transpose(perm = var_1229_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1233, x = var_1229)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 6, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1269,132 +1122,132 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_1453 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_37)[name = tensor("linear_38")]; - tensor var_1454 = const()[name = tensor("op_1454"), val = tensor([6, 2, 4, 64])]; - tensor var_1455 = reshape(shape = var_1454, x = var_1453)[name = tensor("op_1455")]; + tensor var_1241 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1242 = const()[name = tensor("op_1242"), val = tensor([6, 2, 4, 64])]; + tensor var_1243 = reshape(shape = var_1242, x = var_1241)[name = tensor("op_1243")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1459 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_37)[name = tensor("linear_39")]; - tensor var_1460 = const()[name = tensor("op_1460"), val = tensor(0x1p-3)]; - tensor var_1461 = mul(x = var_1459, y = var_1460)[name = tensor("op_1461")]; - tensor var_1462 = const()[name = tensor("op_1462"), val = tensor([6, 2, 4, 64])]; - tensor var_1463 = reshape(shape = var_1462, x = var_1461)[name = tensor("op_1463")]; + tensor var_1247 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1248 = const()[name = tensor("op_1248"), val = tensor(0x1p-3)]; + tensor var_1249 = mul(x = var_1247, y = var_1248)[name = tensor("op_1249")]; + tensor var_1250 = const()[name = tensor("op_1250"), val = tensor([6, 2, 4, 64])]; + tensor var_1251 = reshape(shape = var_1250, x = var_1249)[name = tensor("op_1251")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1467 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_37)[name = tensor("linear_40")]; - tensor var_1468 = const()[name = tensor("op_1468"), val = tensor([6, 2, 4, 64])]; - tensor var_1469 = reshape(shape = var_1468, x = var_1467)[name = tensor("op_1469")]; + tensor var_1255 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1256 = const()[name = tensor("op_1256"), val = tensor([6, 2, 4, 64])]; + tensor var_1257 = reshape(shape = var_1256, x = var_1255)[name = tensor("op_1257")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_209 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_37)[name = tensor("linear_41")]; + tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_1371, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1159, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; - tensor const_24 = const()[name = tensor("const_24"), val = tensor(0x1.fffffep+127)]; - tensor clip_17 = clip(alpha = var_1362, beta = const_24, x = s_t_9)[name = tensor("clip_17")]; + tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; + tensor clip_17 = clip(alpha = var_1149, beta = const_20, x = s_t_9)[name = tensor("clip_17")]; tensor sqrt_s_t_9 = sqrt(x = clip_17)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_1463)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_1455)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1251)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1243)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_1481 = const()[name = tensor("op_1481"), val = tensor([1, 2])]; - tensor var_1482 = reshape(shape = var_1481, x = valid_mask)[name = tensor("op_1482")]; - tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1482)[name = tensor("causal_with_valid_1")]; - tensor var_1484 = const()[name = tensor("op_1484"), val = tensor([2, 1])]; - tensor var_1485 = reshape(shape = var_1484, x = sqrt_s_t_9)[name = tensor("op_1485")]; - tensor M_9 = real_div(x = causal_with_valid_1, y = var_1485)[name = tensor("M_9")]; - tensor var_1487 = mul(x = qk_9, y = M_9)[name = tensor("op_1487")]; + tensor var_1269 = const()[name = tensor("op_1269"), val = tensor([1, 2])]; + tensor var_1270 = reshape(shape = var_1269, x = valid_mask)[name = tensor("op_1270")]; + tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1270)[name = tensor("causal_with_valid_1")]; + tensor var_1272 = const()[name = tensor("op_1272"), val = tensor([2, 1])]; + tensor var_1273 = reshape(shape = var_1272, x = sqrt_s_t_9)[name = tensor("op_1273")]; + tensor M_9 = real_div(x = causal_with_valid_1, y = var_1273)[name = tensor("M_9")]; + tensor var_1275 = mul(x = qk_9, y = M_9)[name = tensor("op_1275")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_1469)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1487, y = v_9)[name = tensor("inner_9")]; - tensor var_1489_transpose_x_0 = const()[name = tensor("op_1489_transpose_x_0"), val = tensor(false)]; - tensor var_1489_transpose_y_0 = const()[name = tensor("op_1489_transpose_y_0"), val = tensor(false)]; - tensor var_1489 = matmul(transpose_x = var_1489_transpose_x_0, transpose_y = var_1489_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1489")]; - tensor var_1490 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1490")]; - tensor var_1491 = const()[name = tensor("op_1491"), val = tensor([1, 1, 2, 1])]; - tensor var_1492 = reshape(shape = var_1491, x = var_1490)[name = tensor("op_1492")]; - tensor cross_9 = mul(x = var_1489, y = var_1492)[name = tensor("cross_9")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1257)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1275, y = v_9)[name = tensor("inner_9")]; + tensor var_1277_transpose_x_0 = const()[name = tensor("op_1277_transpose_x_0"), val = tensor(false)]; + tensor var_1277_transpose_y_0 = const()[name = tensor("op_1277_transpose_y_0"), val = tensor(false)]; + tensor var_1277 = matmul(transpose_x = var_1277_transpose_x_0, transpose_y = var_1277_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1277")]; + tensor var_1278 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1278")]; + tensor var_1279 = const()[name = tensor("op_1279"), val = tensor([1, 1, 2, 1])]; + tensor var_1280 = reshape(shape = var_1279, x = var_1278)[name = tensor("op_1280")]; + tensor cross_9 = mul(x = var_1277, y = var_1280)[name = tensor("cross_9")]; tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; - tensor var_1495 = const()[name = tensor("op_1495"), val = tensor([1, 1, 2, 1])]; - tensor var_1496 = reshape(shape = var_1495, x = valid_mask)[name = tensor("op_1496")]; - tensor v_masked_1 = mul(x = v_9, y = var_1496)[name = tensor("v_masked_1")]; - tensor var_1498 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1498")]; - tensor var_1500_transpose_x_1 = const()[name = tensor("op_1500_transpose_x_1"), val = tensor(true)]; - tensor var_1500_transpose_y_1 = const()[name = tensor("op_1500_transpose_y_1"), val = tensor(false)]; - tensor var_1500 = matmul(transpose_x = var_1500_transpose_x_1, transpose_y = var_1500_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1500")]; - tensor new_kv_unnorm_9 = add(x = var_1498, y = var_1500)[name = tensor("new_kv_unnorm_9")]; - tensor var_1502_keep_dims_0 = const()[name = tensor("op_1502_keep_dims_0"), val = tensor(false)]; - tensor var_1502 = reduce_sum(keep_dims = var_1502_keep_dims_0, x = valid_mask)[name = tensor("op_1502")]; - tensor var_1503 = const()[name = tensor("op_1503"), val = tensor([1])]; - tensor var_1504 = reshape(shape = var_1503, x = var_1502)[name = tensor("op_1504")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_1504)[name = tensor("new_scale_9")]; - tensor const_25 = const()[name = tensor("const_25"), val = tensor(0x1.fffffep+127)]; - tensor clip_18 = clip(alpha = var_1362, beta = const_25, x = new_scale_9)[name = tensor("clip_18")]; + tensor var_1283 = const()[name = tensor("op_1283"), val = tensor([1, 1, 2, 1])]; + tensor var_1284 = reshape(shape = var_1283, x = valid_mask)[name = tensor("op_1284")]; + tensor v_masked_1 = mul(x = v_9, y = var_1284)[name = tensor("v_masked_1")]; + tensor var_1286 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1286")]; + tensor var_1288_transpose_x_1 = const()[name = tensor("op_1288_transpose_x_1"), val = tensor(true)]; + tensor var_1288_transpose_y_1 = const()[name = tensor("op_1288_transpose_y_1"), val = tensor(false)]; + tensor var_1288 = matmul(transpose_x = var_1288_transpose_x_1, transpose_y = var_1288_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1288")]; + tensor new_kv_unnorm_9 = add(x = var_1286, y = var_1288)[name = tensor("new_kv_unnorm_9")]; + tensor var_1290_keep_dims_0 = const()[name = tensor("op_1290_keep_dims_0"), val = tensor(false)]; + tensor var_1290 = reduce_sum(keep_dims = var_1290_keep_dims_0, x = valid_mask)[name = tensor("op_1290")]; + tensor var_1291 = const()[name = tensor("op_1291"), val = tensor([1])]; + tensor var_1292 = reshape(shape = var_1291, x = var_1290)[name = tensor("op_1292")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1292)[name = tensor("new_scale_9")]; + tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; + tensor clip_18 = clip(alpha = var_1149, beta = const_21, x = new_scale_9)[name = tensor("clip_18")]; tensor sqrt_new_scale_1 = sqrt(x = clip_18)[name = tensor("sqrt_new_scale_1")]; - tensor var_1508 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1508")]; - tensor var_1509_perm_0 = const()[name = tensor("op_1509_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1296 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1296")]; + tensor var_1297_perm_0 = const()[name = tensor("op_1297_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_1509 = transpose(perm = var_1509_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1359, x = var_1509)[name = tensor("out_27")]; - tensor var_1513 = const()[name = tensor("op_1513"), val = tensor([6, 2, 256])]; - tensor out_29 = reshape(shape = var_1513, x = out_27)[name = tensor("out_29")]; - tensor var_1515 = silu(x = input_209)[name = tensor("op_1515")]; - tensor input_211 = mul(x = var_1515, y = out_29)[name = tensor("input_211")]; - tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_211)[name = tensor("linear_42")]; - tensor input_213 = add(x = x_37, y = ret_out_9)[name = tensor("input_213")]; + tensor var_1297 = transpose(perm = var_1297_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1146, x = var_1297)[name = tensor("out_27")]; + tensor var_1301 = const()[name = tensor("op_1301"), val = tensor([6, 2, 256])]; + tensor out_29 = reshape(shape = var_1301, x = out_27)[name = tensor("out_29")]; + tensor var_1303 = silu(x = input_169)[name = tensor("op_1303")]; + tensor input_171 = mul(x = var_1303, y = out_29)[name = tensor("input_171")]; + tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; + tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1357, gamma = decoder_norm11_0_weight, x = input_213)[name = tensor("xt_1")]; - tensor var_1525 = const()[name = tensor("op_1525"), val = tensor([1, 6, 2, 256])]; - tensor var_1526 = reshape(shape = var_1525, x = xt_1)[name = tensor("op_1526")]; - tensor var_1527_perm_0 = const()[name = tensor("op_1527_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1530 = const()[name = tensor("op_1530"), val = tensor([2, 6, 256])]; - tensor var_1527 = transpose(perm = var_1527_perm_0, x = var_1526)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_1530, x = var_1527)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1144, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1313 = const()[name = tensor("op_1313"), val = tensor([1, 6, 2, 256])]; + tensor var_1314 = reshape(shape = var_1313, x = xt_1)[name = tensor("op_1314")]; + tensor var_1315_perm_0 = const()[name = tensor("op_1315_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1318 = const()[name = tensor("op_1318"), val = tensor([2, 6, 256])]; + tensor var_1315 = transpose(perm = var_1315_perm_0, x = var_1314)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1318, x = var_1315)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_1553 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_8 = const()[name = tensor("concat_8"), val = tensor([6, 2, 3, 256])]; - tensor var_1555 = reshape(shape = concat_8, x = var_1553)[name = tensor("op_1555")]; - tensor var_1556_axes_0 = const()[name = tensor("op_1556_axes_0"), val = tensor([0])]; - tensor var_1556 = expand_dims(axes = var_1556_axes_0, x = var_1555)[name = tensor("op_1556")]; - tensor var_1557_perm_0 = const()[name = tensor("op_1557_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1558_axes_0 = const()[name = tensor("op_1558_axes_0"), val = tensor([-2])]; - tensor var_1557 = transpose(perm = var_1557_perm_0, x = var_1556)[name = tensor("transpose_21")]; - tensor var_1558 = squeeze(axes = var_1558_axes_0, x = var_1557)[name = tensor("op_1558")]; + tensor var_1341 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_9 = const()[name = tensor("concat_9"), val = tensor([6, 2, 3, 256])]; + tensor var_1343 = reshape(shape = concat_9, x = var_1341)[name = tensor("op_1343")]; + tensor var_1344_axes_0 = const()[name = tensor("op_1344_axes_0"), val = tensor([0])]; + tensor var_1344 = expand_dims(axes = var_1344_axes_0, x = var_1343)[name = tensor("op_1344")]; + tensor var_1345_perm_0 = const()[name = tensor("op_1345_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1346_axes_0 = const()[name = tensor("op_1346_axes_0"), val = tensor([-2])]; + tensor var_1345 = transpose(perm = var_1345_perm_0, x = var_1344)[name = tensor("transpose_21")]; + tensor var_1346 = squeeze(axes = var_1346_axes_0, x = var_1345)[name = tensor("op_1346")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 6, 2, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1558)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1346)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 6, 2, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1558)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1346)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 6, 2, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1558)[name = tensor("v_11")]; - tensor var_1566 = const()[name = tensor("op_1566"), val = tensor([6, 8, 64])]; - tensor var_1567 = reshape(shape = var_1566, x = q_11)[name = tensor("op_1567")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1346)[name = tensor("v_11")]; + tensor var_1354 = const()[name = tensor("op_1354"), val = tensor([6, 8, 64])]; + tensor var_1355 = reshape(shape = var_1354, x = q_11)[name = tensor("op_1355")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1573 = const()[name = tensor("op_1573"), val = tensor([6, 8, 64])]; - tensor var_1574 = reshape(shape = var_1573, x = k_11)[name = tensor("op_1574")]; + tensor var_1361 = const()[name = tensor("op_1361"), val = tensor([6, 8, 64])]; + tensor var_1362 = reshape(shape = var_1361, x = k_11)[name = tensor("op_1362")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1580 = const()[name = tensor("op_1580"), val = tensor([6, 8, 64])]; - tensor var_1581 = reshape(shape = var_1580, x = v_11)[name = tensor("op_1581")]; + tensor var_1368 = const()[name = tensor("op_1368"), val = tensor([6, 8, 64])]; + tensor var_1369 = reshape(shape = var_1368, x = v_11)[name = tensor("op_1369")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1584 = const()[name = tensor("op_1584"), val = tensor([2, 4, 6, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_1567)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_1584, x = q_13)[name = tensor("q_15")]; - tensor var_1586 = const()[name = tensor("op_1586"), val = tensor([2, 4, 6, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_1574)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_1586, x = k_13)[name = tensor("k_15")]; - tensor var_1588 = const()[name = tensor("op_1588"), val = tensor([2, 4, 6, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_1581)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_1588, x = v_13)[name = tensor("v_15")]; + tensor var_1372 = const()[name = tensor("op_1372"), val = tensor([2, 4, 6, 64])]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1355)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1372, x = q_13)[name = tensor("q_15")]; + tensor var_1374 = const()[name = tensor("op_1374"), val = tensor([2, 4, 6, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1362)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1374, x = k_13)[name = tensor("k_15")]; + tensor var_1376 = const()[name = tensor("op_1376"), val = tensor([2, 4, 6, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1369)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1376, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -1405,30 +1258,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_1591 = const()[name = tensor("op_1591"), val = tensor([2, 0, 1, 3])]; - tensor var_1596 = const()[name = tensor("op_1596"), val = tensor([12, 256])]; - tensor var_1592 = transpose(perm = var_1591, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_1596, x = var_1592)[name = tensor("attn_output_3")]; + tensor var_1379 = const()[name = tensor("op_1379"), val = tensor([2, 0, 1, 3])]; + tensor var_1384 = const()[name = tensor("op_1384"), val = tensor([12, 256])]; + tensor var_1380 = transpose(perm = var_1379, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1384, x = var_1380)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_1600 = const()[name = tensor("op_1600"), val = tensor([6, 2, 256])]; - tensor attn_output_7 = reshape(shape = var_1600, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1388 = const()[name = tensor("op_1388"), val = tensor([6, 2, 256])]; + tensor attn_output_7 = reshape(shape = var_1388, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; - tensor input_215 = add(x = query_1, y = sa2_out_1)[name = tensor("input_215")]; - tensor input_217_axes_0 = const()[name = tensor("input_217_axes_0"), val = tensor([-1])]; - tensor input_217 = layer_norm(axes = input_217_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1357, gamma = decoder_norm21_0_weight, x = input_215)[name = tensor("input_217")]; - tensor input_219 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_217)[name = tensor("linear_45")]; - tensor input_221 = relu(x = input_219)[name = tensor("input_221")]; - tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_221)[name = tensor("linear_46")]; - tensor input_223 = add(x = input_217, y = ffn_out_1)[name = tensor("input_223")]; + tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; + tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1144, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; + tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; + tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; + tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1357, gamma = decoder_norm22_0_weight, x = input_223)[name = tensor("xt_3")]; - tensor var_1620 = const()[name = tensor("op_1620"), val = tensor([1, 2, 6, 256])]; - tensor x_39 = reshape(shape = var_1620, x = xt_3)[name = tensor("x_39")]; - tensor var_1622_perm_0 = const()[name = tensor("op_1622_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1626 = const()[name = tensor("op_1626"), val = tensor([6, 2, 256])]; - tensor var_1622 = transpose(perm = var_1622_perm_0, x = x_39)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_1626, x = var_1622)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1144, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1408 = const()[name = tensor("op_1408"), val = tensor([1, 2, 6, 256])]; + tensor x_31 = reshape(shape = var_1408, x = xt_3)[name = tensor("x_31")]; + tensor var_1410_perm_0 = const()[name = tensor("op_1410_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1414 = const()[name = tensor("op_1414"), val = tensor([6, 2, 256])]; + tensor var_1410 = transpose(perm = var_1410_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1414, x = var_1410)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 6, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1439,120 +1292,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_1634 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_1635 = const()[name = tensor("op_1635"), val = tensor([6, 2, 4, 64])]; - tensor var_1636 = reshape(shape = var_1635, x = var_1634)[name = tensor("op_1636")]; + tensor var_1422 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1423 = const()[name = tensor("op_1423"), val = tensor([6, 2, 4, 64])]; + tensor var_1424 = reshape(shape = var_1423, x = var_1422)[name = tensor("op_1424")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1640 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_1641 = const()[name = tensor("op_1641"), val = tensor(0x1p-3)]; - tensor var_1642 = mul(x = var_1640, y = var_1641)[name = tensor("op_1642")]; - tensor var_1643 = const()[name = tensor("op_1643"), val = tensor([6, 2, 4, 64])]; - tensor var_1644 = reshape(shape = var_1643, x = var_1642)[name = tensor("op_1644")]; + tensor var_1428 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1429 = const()[name = tensor("op_1429"), val = tensor(0x1p-3)]; + tensor var_1430 = mul(x = var_1428, y = var_1429)[name = tensor("op_1430")]; + tensor var_1431 = const()[name = tensor("op_1431"), val = tensor([6, 2, 4, 64])]; + tensor var_1432 = reshape(shape = var_1431, x = var_1430)[name = tensor("op_1432")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1648 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_1649 = const()[name = tensor("op_1649"), val = tensor([6, 2, 4, 64])]; - tensor var_1650 = reshape(shape = var_1649, x = var_1648)[name = tensor("op_1650")]; + tensor var_1436 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1437 = const()[name = tensor("op_1437"), val = tensor([6, 2, 4, 64])]; + tensor var_1438 = reshape(shape = var_1437, x = var_1436)[name = tensor("op_1438")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_227 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; + tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; - tensor const_36 = const()[name = tensor("const_36"), val = tensor(0x1.fffffep+127)]; - tensor clip_19 = clip(alpha = var_1362, beta = const_36, x = s_t)[name = tensor("clip_19")]; + tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; + tensor clip_19 = clip(alpha = var_1149, beta = const_32, x = s_t)[name = tensor("clip_19")]; tensor sqrt_s_t = sqrt(x = clip_19)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_1644)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_1636)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1432)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1424)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_1665 = const()[name = tensor("op_1665"), val = tensor([2, 1])]; - tensor var_1666 = reshape(shape = var_1665, x = sqrt_s_t)[name = tensor("op_1666")]; - tensor M = real_div(x = causal_with_valid_1, y = var_1666)[name = tensor("M")]; - tensor var_1668 = mul(x = qk, y = M)[name = tensor("op_1668")]; + tensor var_1453 = const()[name = tensor("op_1453"), val = tensor([2, 1])]; + tensor var_1454 = reshape(shape = var_1453, x = sqrt_s_t)[name = tensor("op_1454")]; + tensor M = real_div(x = causal_with_valid_1, y = var_1454)[name = tensor("M")]; + tensor var_1456 = mul(x = qk, y = M)[name = tensor("op_1456")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_1650)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1668, y = v_17)[name = tensor("inner")]; - tensor var_1670_transpose_x_0 = const()[name = tensor("op_1670_transpose_x_0"), val = tensor(false)]; - tensor var_1670_transpose_y_0 = const()[name = tensor("op_1670_transpose_y_0"), val = tensor(false)]; - tensor var_1670 = matmul(transpose_x = var_1670_transpose_x_0, transpose_y = var_1670_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1670")]; - tensor var_1671 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1671")]; - tensor var_1672 = const()[name = tensor("op_1672"), val = tensor([1, 1, 2, 1])]; - tensor var_1673 = reshape(shape = var_1672, x = var_1671)[name = tensor("op_1673")]; - tensor cross = mul(x = var_1670, y = var_1673)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1438)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1456, y = v_17)[name = tensor("inner")]; + tensor var_1458_transpose_x_0 = const()[name = tensor("op_1458_transpose_x_0"), val = tensor(false)]; + tensor var_1458_transpose_y_0 = const()[name = tensor("op_1458_transpose_y_0"), val = tensor(false)]; + tensor var_1458 = matmul(transpose_x = var_1458_transpose_x_0, transpose_y = var_1458_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1458")]; + tensor var_1459 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1459")]; + tensor var_1460 = const()[name = tensor("op_1460"), val = tensor([1, 1, 2, 1])]; + tensor var_1461 = reshape(shape = var_1460, x = var_1459)[name = tensor("op_1461")]; + tensor cross = mul(x = var_1458, y = var_1461)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_1496)[name = tensor("v_masked")]; - tensor var_1679 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1679")]; - tensor var_1681_transpose_x_1 = const()[name = tensor("op_1681_transpose_x_1"), val = tensor(true)]; - tensor var_1681_transpose_y_1 = const()[name = tensor("op_1681_transpose_y_1"), val = tensor(false)]; - tensor var_1681 = matmul(transpose_x = var_1681_transpose_x_1, transpose_y = var_1681_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1681")]; - tensor new_kv_unnorm = add(x = var_1679, y = var_1681)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_1504)[name = tensor("new_scale")]; - tensor const_37 = const()[name = tensor("const_37"), val = tensor(0x1.fffffep+127)]; - tensor clip_20 = clip(alpha = var_1362, beta = const_37, x = new_scale)[name = tensor("clip_20")]; + tensor v_masked = mul(x = v_17, y = var_1284)[name = tensor("v_masked")]; + tensor var_1467 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1467")]; + tensor var_1469_transpose_x_1 = const()[name = tensor("op_1469_transpose_x_1"), val = tensor(true)]; + tensor var_1469_transpose_y_1 = const()[name = tensor("op_1469_transpose_y_1"), val = tensor(false)]; + tensor var_1469 = matmul(transpose_x = var_1469_transpose_x_1, transpose_y = var_1469_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1469")]; + tensor new_kv_unnorm = add(x = var_1467, y = var_1469)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1292)[name = tensor("new_scale")]; + tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; + tensor clip_20 = clip(alpha = var_1149, beta = const_33, x = new_scale)[name = tensor("clip_20")]; tensor sqrt_new_scale = sqrt(x = clip_20)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_1690_perm_0 = const()[name = tensor("op_1690_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1478_perm_0 = const()[name = tensor("op_1478_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_1690 = transpose(perm = var_1690_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1359, x = var_1690)[name = tensor("out_33")]; - tensor var_1694 = const()[name = tensor("op_1694"), val = tensor([6, 2, 256])]; - tensor out = reshape(shape = var_1694, x = out_33)[name = tensor("out")]; - tensor var_1696 = silu(x = input_227)[name = tensor("op_1696")]; - tensor input_229 = mul(x = var_1696, y = out)[name = tensor("input_229")]; - tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_229)[name = tensor("linear_51")]; - tensor input_231 = add(x = x, y = ret_out)[name = tensor("input_231")]; + tensor var_1478 = transpose(perm = var_1478_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1146, x = var_1478)[name = tensor("out_33")]; + tensor var_1482 = const()[name = tensor("op_1482"), val = tensor([6, 2, 256])]; + tensor out = reshape(shape = var_1482, x = out_33)[name = tensor("out")]; + tensor var_1484 = silu(x = input_187)[name = tensor("op_1484")]; + tensor input_189 = mul(x = var_1484, y = out)[name = tensor("input_189")]; + tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; + tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1357, gamma = decoder_norm11_1_weight, x = input_231)[name = tensor("xt_5")]; - tensor var_1706 = const()[name = tensor("op_1706"), val = tensor([1, 6, 2, 256])]; - tensor var_1707 = reshape(shape = var_1706, x = xt_5)[name = tensor("op_1707")]; - tensor var_1708_perm_0 = const()[name = tensor("op_1708_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1711 = const()[name = tensor("op_1711"), val = tensor([2, 6, 256])]; - tensor var_1708 = transpose(perm = var_1708_perm_0, x = var_1707)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_1711, x = var_1708)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1144, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1494 = const()[name = tensor("op_1494"), val = tensor([1, 6, 2, 256])]; + tensor var_1495 = reshape(shape = var_1494, x = xt_5)[name = tensor("op_1495")]; + tensor var_1496_perm_0 = const()[name = tensor("op_1496_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1499 = const()[name = tensor("op_1499"), val = tensor([2, 6, 256])]; + tensor var_1496 = transpose(perm = var_1496_perm_0, x = var_1495)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1499, x = var_1496)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_1734 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_9 = const()[name = tensor("concat_9"), val = tensor([6, 2, 3, 256])]; - tensor var_1736 = reshape(shape = concat_9, x = var_1734)[name = tensor("op_1736")]; - tensor var_1737_axes_0 = const()[name = tensor("op_1737_axes_0"), val = tensor([0])]; - tensor var_1737 = expand_dims(axes = var_1737_axes_0, x = var_1736)[name = tensor("op_1737")]; - tensor var_1738_perm_0 = const()[name = tensor("op_1738_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1739_axes_0 = const()[name = tensor("op_1739_axes_0"), val = tensor([-2])]; - tensor var_1738 = transpose(perm = var_1738_perm_0, x = var_1737)[name = tensor("transpose_8")]; - tensor var_1739 = squeeze(axes = var_1739_axes_0, x = var_1738)[name = tensor("op_1739")]; + tensor var_1522 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_10 = const()[name = tensor("concat_10"), val = tensor([6, 2, 3, 256])]; + tensor var_1524 = reshape(shape = concat_10, x = var_1522)[name = tensor("op_1524")]; + tensor var_1525_axes_0 = const()[name = tensor("op_1525_axes_0"), val = tensor([0])]; + tensor var_1525 = expand_dims(axes = var_1525_axes_0, x = var_1524)[name = tensor("op_1525")]; + tensor var_1526_perm_0 = const()[name = tensor("op_1526_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1527_axes_0 = const()[name = tensor("op_1527_axes_0"), val = tensor([-2])]; + tensor var_1526 = transpose(perm = var_1526_perm_0, x = var_1525)[name = tensor("transpose_8")]; + tensor var_1527 = squeeze(axes = var_1527_axes_0, x = var_1526)[name = tensor("op_1527")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 6, 2, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1739)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1527)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 6, 2, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1739)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1527)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 6, 2, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1739)[name = tensor("v_19")]; - tensor var_1747 = const()[name = tensor("op_1747"), val = tensor([6, 8, 64])]; - tensor var_1748 = reshape(shape = var_1747, x = q_19)[name = tensor("op_1748")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1527)[name = tensor("v_19")]; + tensor var_1535 = const()[name = tensor("op_1535"), val = tensor([6, 8, 64])]; + tensor var_1536 = reshape(shape = var_1535, x = q_19)[name = tensor("op_1536")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1754 = const()[name = tensor("op_1754"), val = tensor([6, 8, 64])]; - tensor var_1755 = reshape(shape = var_1754, x = k_19)[name = tensor("op_1755")]; + tensor var_1542 = const()[name = tensor("op_1542"), val = tensor([6, 8, 64])]; + tensor var_1543 = reshape(shape = var_1542, x = k_19)[name = tensor("op_1543")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1761 = const()[name = tensor("op_1761"), val = tensor([6, 8, 64])]; - tensor var_1762 = reshape(shape = var_1761, x = v_19)[name = tensor("op_1762")]; + tensor var_1549 = const()[name = tensor("op_1549"), val = tensor([6, 8, 64])]; + tensor var_1550 = reshape(shape = var_1549, x = v_19)[name = tensor("op_1550")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1765 = const()[name = tensor("op_1765"), val = tensor([2, 4, 6, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_1748)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_1765, x = q_21)[name = tensor("q")]; - tensor var_1767 = const()[name = tensor("op_1767"), val = tensor([2, 4, 6, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_1755)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_1767, x = k_21)[name = tensor("k")]; - tensor var_1769 = const()[name = tensor("op_1769"), val = tensor([2, 4, 6, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_1762)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_1769, x = v_21)[name = tensor("v")]; + tensor var_1553 = const()[name = tensor("op_1553"), val = tensor([2, 4, 6, 64])]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1536)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1553, x = q_21)[name = tensor("q")]; + tensor var_1555 = const()[name = tensor("op_1555"), val = tensor([2, 4, 6, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1543)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1555, x = k_21)[name = tensor("k")]; + tensor var_1557 = const()[name = tensor("op_1557"), val = tensor([2, 4, 6, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1550)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1557, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -1563,50 +1416,50 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_1772 = const()[name = tensor("op_1772"), val = tensor([2, 0, 1, 3])]; - tensor var_1777 = const()[name = tensor("op_1777"), val = tensor([12, 256])]; - tensor var_1773 = transpose(perm = var_1772, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_1777, x = var_1773)[name = tensor("attn_output_11")]; + tensor var_1560 = const()[name = tensor("op_1560"), val = tensor([2, 0, 1, 3])]; + tensor var_1565 = const()[name = tensor("op_1565"), val = tensor([12, 256])]; + tensor var_1561 = transpose(perm = var_1560, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1565, x = var_1561)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_1781 = const()[name = tensor("op_1781"), val = tensor([6, 2, 256])]; - tensor attn_output = reshape(shape = var_1781, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1569 = const()[name = tensor("op_1569"), val = tensor([6, 2, 256])]; + tensor attn_output = reshape(shape = var_1569, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; - tensor input_233 = add(x = query_5, y = sa2_out)[name = tensor("input_233")]; - tensor input_235_axes_0 = const()[name = tensor("input_235_axes_0"), val = tensor([-1])]; - tensor input_235 = layer_norm(axes = input_235_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1357, gamma = decoder_norm21_1_weight, x = input_233)[name = tensor("input_235")]; - tensor input_237 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_235)[name = tensor("linear_54")]; - tensor input_239 = relu(x = input_237)[name = tensor("input_239")]; - tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_239)[name = tensor("linear_55")]; - tensor input_241 = add(x = input_235, y = ffn_out)[name = tensor("input_241")]; + tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; + tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1144, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; + tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; + tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; + tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1357, gamma = decoder_norm22_1_weight, x = input_241)[name = tensor("xt")]; - tensor var_1801 = const()[name = tensor("op_1801"), val = tensor([1, 2, 6, 256])]; - tensor input = reshape(shape = var_1801, x = xt)[name = tensor("input")]; - tensor var_1803 = const()[name = tensor("op_1803"), val = tensor([-1])]; - tensor var_1804 = reduce_l2_norm(axes = var_1803, keep_dims = var_1360, x = input)[name = tensor("op_1804")]; - tensor const_46 = const()[name = tensor("const_46"), val = tensor(0x1.fffffep+127)]; - tensor clip_21 = clip(alpha = var_1351, beta = const_46, x = var_1804)[name = tensor("clip_21")]; - tensor var_1806 = real_div(x = input, y = clip_21)[name = tensor("op_1806")]; - tensor concat_13 = const()[name = tensor("concat_13"), val = tensor([2, 1, 256])]; - tensor reshape_0 = reshape(shape = concat_13, x = emb)[name = tensor("reshape_0")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1144, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1589 = const()[name = tensor("op_1589"), val = tensor([1, 2, 6, 256])]; + tensor input = reshape(shape = var_1589, x = xt)[name = tensor("input")]; + tensor var_1591 = const()[name = tensor("op_1591"), val = tensor([-1])]; + tensor var_1592 = reduce_l2_norm(axes = var_1591, keep_dims = var_1147, x = input)[name = tensor("op_1592")]; + tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; + tensor clip_21 = clip(alpha = var_1139, beta = const_42, x = var_1592)[name = tensor("clip_21")]; + tensor var_1594 = real_div(x = input, y = clip_21)[name = tensor("op_1594")]; + tensor concat_14 = const()[name = tensor("concat_14"), val = tensor([2, 1, 256])]; + tensor reshape_0 = reshape(shape = concat_14, x = emb)[name = tensor("reshape_0")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_14 = const()[name = tensor("concat_14"), val = tensor([2, 256, 6])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1806)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_14, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_15 = const()[name = tensor("concat_15"), val = tensor([2, 256, 6])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1594)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_15, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = reshape_0, y = reshape_1)[name = tensor("matmul_2")]; - tensor concat_17 = const()[name = tensor("concat_17"), val = tensor([1, 2, 6])]; - tensor reshape_2 = reshape(shape = concat_17, x = matmul_2)[name = tensor("reshape_2")]; + tensor concat_18 = const()[name = tensor("concat_18"), val = tensor([1, 2, 6])]; + tensor reshape_2 = reshape(shape = concat_18, x = matmul_2)[name = tensor("reshape_2")]; tensor output_begin_0 = const()[name = tensor("output_begin_0"), val = tensor([0, 0, 1])]; tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 2, 5])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = reshape_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_1810")]; - tensor var_1812_axis_0 = const()[name = tensor("op_1812_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_1812_axis_0, values = (var_1508, nkv))[name = tensor("op_1812")]; - tensor var_1814_axis_0 = const()[name = tensor("op_1814_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_1814_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1814")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1598")]; + tensor var_1600_axis_0 = const()[name = tensor("op_1600_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1600_axis_0, values = (var_1296, nkv))[name = tensor("op_1600")]; + tensor var_1602_axis_0 = const()[name = tensor("op_1602_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1602_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1602")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/AMI/ls_eend_ami_200ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/AMI/ls_eend_ami_200ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index f4428daed050de3787555c22df831570054ce95d..2205df025b0a92b24d412143180794b971c5d1d0 100644 --- a/optimized/AMI/ls_eend_ami_200ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/AMI/ls_eend_ami_200ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3293b85aef8108cf8b1663a4e971e01752330cf574c1b058aac0c84c99330617 -size 228657 +oid sha256:2acf302970eda1aefaafa30730b9903db30b2a0bde26262532c58c2de5966aa3 +size 205508 diff --git a/optimized/AMI/ls_eend_ami_200ms.mlpackage/Manifest.json b/optimized/AMI/ls_eend_ami_200ms.mlpackage/Manifest.json index 2726e0c2ea92fec8a0cf5569d88f5a0536f1f2ca..bd97ca3aabcaf84203f7ae07c3d67e812fdadb16 100644 --- a/optimized/AMI/ls_eend_ami_200ms.mlpackage/Manifest.json +++ b/optimized/AMI/ls_eend_ami_200ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "1CED9BE1-0926-4A69-95D3-F47806E90E84": { + "1C9AE01B-EEE1-4BE2-9A50-C27B9CF3B516": { "author": "com.apple.CoreML", "description": "CoreML Model Weights", "name": "weights", "path": "com.apple.CoreML/weights" }, - "2FEDB331-DC2D-4274-8B0A-9533FF25053F": { + "79B1F9E3-1044-4EFB-BBA9-DE9E200A8049": { "author": "com.apple.CoreML", "description": "CoreML Model Specification", "name": "model.mlmodel", "path": "com.apple.CoreML/model.mlmodel" } }, - "rootModelIdentifier": "2FEDB331-DC2D-4274-8B0A-9533FF25053F" + "rootModelIdentifier": "79B1F9E3-1044-4EFB-BBA9-DE9E200A8049" } diff --git a/optimized/AMI/ls_eend_ami_300ms.mlmodelc/analytics/coremldata.bin b/optimized/AMI/ls_eend_ami_300ms.mlmodelc/analytics/coremldata.bin index 99a2df3a98f79f662728ab6a1bcf275589f02eee..2077109adcc75bd02c783586443e8078e2a6a935 100644 --- a/optimized/AMI/ls_eend_ami_300ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/AMI/ls_eend_ami_300ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:29af106d70c27fee8bf3f2bc4125663994057125f639325ccd84851405ce15ff +oid sha256:41d15a95f310c21630f37db1665657b38819a80da5dfdd51fb3c9de331d617c0 size 243 diff --git a/optimized/AMI/ls_eend_ami_300ms.mlmodelc/coremldata.bin b/optimized/AMI/ls_eend_ami_300ms.mlmodelc/coremldata.bin index cba371c6ec65c4fbcca3fdbd16f3d9d540d327d8..608324b0333559029c9b0d018cbcc8910a169348 100644 --- a/optimized/AMI/ls_eend_ami_300ms.mlmodelc/coremldata.bin +++ b/optimized/AMI/ls_eend_ami_300ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4e99b889d2b58ca355f6d0d24e50d8abc5d161e846db6f9ca550093628f24751 -size 1250 +oid sha256:ba9404169108b7fd9062e51e08715cbb18e178a87f79ef47351497b94d658c6b +size 1341 diff --git a/optimized/AMI/ls_eend_ami_300ms.mlmodelc/metadata.json b/optimized/AMI/ls_eend_ami_300ms.mlmodelc/metadata.json index 3fe2bc8fb0889cabd57006ba8ad105041d2350f2..8861010bf7cbab225883b14e81bd8da5789ffc78 100644 --- a/optimized/AMI/ls_eend_ami_300ms.mlmodelc/metadata.json +++ b/optimized/AMI/ls_eend_ami_300ms.mlmodelc/metadata.json @@ -96,30 +96,30 @@ "Ios17.reshape" : 92, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 69, - "Split" : 12, + "Ios17.transpose" : 57, + "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 82, "Tile" : 13, - "Ios17.sliceByIndex" : 76, - "Ios16.sigmoid" : 13, + "Ios17.sliceByIndex" : 60, + "Ios16.sigmoid" : 5, "Ios16.reduceSum" : 1, "Ios17.squeeze" : 2, - "Ios17.layerNorm" : 41, - "Ios17.batchNorm" : 12, + "Ios17.layerNorm" : 33, + "Ios17.batchNorm" : 4, "Ios17.cast" : 24, "Ios17.sqrt" : 18, "Ios17.sub" : 24, - "Ios17.conv" : 37, + "Ios17.conv" : 13, "Ios17.realDiv" : 20, "Ios17.linear" : 56, "Ios17.concat" : 18, "Ios17.clip" : 30, "Ios17.greaterEqual" : 12, - "Ios16.silu" : 26, + "Ios16.silu" : 18, "Stack" : 6, "Ios16.cumsum" : 1, - "Ios17.mul" : 103 + "Ios17.mul" : 95 }, "computePrecision" : "Mixed (Float32, Int32)", "isUpdatable" : "0", @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"ami\", \"model_label\": \"AMI\", \"variant\": \"pipeline\", \"T\": 3, \"max_speakers\": 4, \"max_nspks\": 6, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"ami\", \"model_label\": \"AMI\", \"variant\": \"pipeline\", \"chunk_size\": 3, \"step_duration_ms\": 300, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 4, \"max_nspks\": 6, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/AMI/ls_eend_ami_300ms.mlmodelc/model.mil b/optimized/AMI/ls_eend_ami_300ms.mlmodelc/model.mil index 5bd751b164eca9d81e212de69fdb649ac6d407a1..0615b049ed1f6d284b1a01ee41a6b6fbff22a498 100644 --- a/optimized/AMI/ls_eend_ami_300ms.mlmodelc/model.mil +++ b/optimized/AMI/ls_eend_ami_300ms.mlmodelc/model.mil @@ -217,6 +217,7 @@ program(1.0) tensor var_21 = const()[name = tensor("op_21"), val = tensor(0x0p+0)]; tensor var_24 = const()[name = tensor("op_24"), val = tensor(0x1.0c6f7ap-20)]; tensor var_27 = const()[name = tensor("op_27"), val = tensor(2)]; + tensor var_30 = const()[name = tensor("op_30"), val = tensor(0)]; tensor var_33 = const()[name = tensor("op_33"), val = tensor(1)]; tensor var_35 = const()[name = tensor("op_35"), val = tensor(0x1.4f8b58p-17)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(true)]; @@ -263,8 +264,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_69")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_70")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_196 = const()[name = tensor("op_196"), val = tensor([3, 1])]; tensor var_197 = reshape(shape = var_196, x = sqrt_s_t_1)[name = tensor("op_197")]; @@ -272,7 +273,7 @@ program(1.0) tensor var_199 = mul(x = qk_1, y = M_1)[name = tensor("op_199")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_68")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_199, y = v_1)[name = tensor("inner_1")]; tensor var_201_transpose_x_0 = const()[name = tensor("op_201_transpose_x_0"), val = tensor(false)]; tensor var_201_transpose_y_0 = const()[name = tensor("op_201_transpose_y_0"), val = tensor(false)]; @@ -293,7 +294,7 @@ program(1.0) tensor var_214 = real_div(x = new_kv_unnorm_1, y = var_213)[name = tensor("op_214")]; tensor var_215_perm_0 = const()[name = tensor("op_215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_67")]; + tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_24, x = var_215)[name = tensor("out_3")]; tensor var_219 = const()[name = tensor("op_219"), val = tensor([1, 3, 256])]; tensor out_5 = reshape(shape = var_219, x = out_3)[name = tensor("out_5")]; @@ -341,218 +342,142 @@ program(1.0) tensor var_252 = sub(x = var_17, y = use_roll_1)[name = tensor("op_252")]; tensor var_253 = mul(x = filled_window_1, y = var_252)[name = tensor("op_253")]; tensor var_254 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_254")]; - tensor input_21 = add(x = var_253, y = var_254)[name = tensor("input_21")]; + tensor window_3 = add(x = var_253, y = var_254)[name = tensor("window_3")]; tensor var_256 = const()[name = tensor("op_256"), val = tensor(0x1p+0)]; tensor var_257 = add(x = filled_1, y = var_256)[name = tensor("op_257")]; tensor clip_1 = clip(alpha = var_21, beta = var_16, x = var_257)[name = tensor("clip_1")]; + tensor var_260_begin_0 = const()[name = tensor("op_260_begin_0"), val = tensor([0, 1, 0])]; + tensor var_260_end_0 = const()[name = tensor("op_260_end_0"), val = tensor([1, 2, 256])]; + tensor var_260_end_mask_0 = const()[name = tensor("op_260_end_mask_0"), val = tensor([true, false, true])]; + tensor var_260 = slice_by_index(begin = var_260_begin_0, end = var_260_end_0, end_mask = var_260_end_mask_0, x = x_3)[name = tensor("op_260")]; + tensor var_263_begin_0 = const()[name = tensor("op_263_begin_0"), val = tensor([0, 1, 0])]; + tensor var_263_end_0 = const()[name = tensor("op_263_end_0"), val = tensor([1, 16, 256])]; + tensor var_263_end_mask_0 = const()[name = tensor("op_263_end_mask_0"), val = tensor([true, true, true])]; + tensor var_263 = slice_by_index(begin = var_263_begin_0, end = var_263_end_0, end_mask = var_263_end_mask_0, x = window_3)[name = tensor("op_263")]; + tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; + tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_263, var_260))[name = tensor("rolled_3")]; + tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; + tensor var_268 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_268")]; + tensor var_269_dtype_0 = const()[name = tensor("op_269_dtype_0"), val = tensor("fp32")]; + tensor var_270 = const()[name = tensor("op_270"), val = tensor([1, 16, 1])]; + tensor var_269 = cast(dtype = var_269_dtype_0, x = var_268)[name = tensor("cast_157")]; + tensor fill_mask_3 = reshape(shape = var_270, x = var_269)[name = tensor("fill_mask_3")]; + tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_260)[name = tensor("x_expanded_3")]; + tensor var_274 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_274")]; + tensor var_275 = mul(x = window_3, y = var_274)[name = tensor("op_275")]; + tensor var_276 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_276")]; + tensor filled_window_3 = add(x = var_275, y = var_276)[name = tensor("filled_window_3")]; + tensor var_278 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_278")]; + tensor var_279_dtype_0 = const()[name = tensor("op_279_dtype_0"), val = tensor("fp32")]; + tensor var_280 = const()[name = tensor("op_280"), val = tensor([1, 1, 1])]; + tensor var_279 = cast(dtype = var_279_dtype_0, x = var_278)[name = tensor("cast_156")]; + tensor use_roll_3 = reshape(shape = var_280, x = var_279)[name = tensor("use_roll_3")]; + tensor var_282 = sub(x = var_17, y = use_roll_3)[name = tensor("op_282")]; + tensor var_283 = mul(x = filled_window_3, y = var_282)[name = tensor("op_283")]; + tensor var_284 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_284")]; + tensor window_5 = add(x = var_283, y = var_284)[name = tensor("window_5")]; + tensor var_286 = const()[name = tensor("op_286"), val = tensor(0x1p+0)]; + tensor var_287 = add(x = clip_1, y = var_286)[name = tensor("op_287")]; + tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_287)[name = tensor("clip_3")]; + tensor var_290_begin_0 = const()[name = tensor("op_290_begin_0"), val = tensor([0, 2, 0])]; + tensor var_290_end_0 = const()[name = tensor("op_290_end_0"), val = tensor([1, 1, 256])]; + tensor var_290_end_mask_0 = const()[name = tensor("op_290_end_mask_0"), val = tensor([true, true, true])]; + tensor var_290 = slice_by_index(begin = var_290_begin_0, end = var_290_end_0, end_mask = var_290_end_mask_0, x = x_3)[name = tensor("op_290")]; + tensor var_293_begin_0 = const()[name = tensor("op_293_begin_0"), val = tensor([0, 1, 0])]; + tensor var_293_end_0 = const()[name = tensor("op_293_end_0"), val = tensor([1, 16, 256])]; + tensor var_293_end_mask_0 = const()[name = tensor("op_293_end_mask_0"), val = tensor([true, true, true])]; + tensor var_293 = slice_by_index(begin = var_293_begin_0, end = var_293_end_0, end_mask = var_293_end_mask_0, x = window_5)[name = tensor("op_293")]; + tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; + tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_293, var_290))[name = tensor("rolled_5")]; + tensor clip_4 = clip(alpha = var_21, beta = var_20, x = clip_3)[name = tensor("clip_4")]; + tensor var_298 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_298")]; + tensor var_299_dtype_0 = const()[name = tensor("op_299_dtype_0"), val = tensor("fp32")]; + tensor var_300 = const()[name = tensor("op_300"), val = tensor([1, 16, 1])]; + tensor var_299 = cast(dtype = var_299_dtype_0, x = var_298)[name = tensor("cast_155")]; + tensor fill_mask_5 = reshape(shape = var_300, x = var_299)[name = tensor("fill_mask_5")]; + tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_290)[name = tensor("x_expanded_5")]; + tensor var_304 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_304")]; + tensor var_305 = mul(x = window_5, y = var_304)[name = tensor("op_305")]; + tensor var_306 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_306")]; + tensor filled_window_5 = add(x = var_305, y = var_306)[name = tensor("filled_window_5")]; + tensor var_308 = greater_equal(x = clip_3, y = var_16)[name = tensor("op_308")]; + tensor var_309_dtype_0 = const()[name = tensor("op_309_dtype_0"), val = tensor("fp32")]; + tensor var_310 = const()[name = tensor("op_310"), val = tensor([1, 1, 1])]; + tensor var_309 = cast(dtype = var_309_dtype_0, x = var_308)[name = tensor("cast_154")]; + tensor use_roll_5 = reshape(shape = var_310, x = var_309)[name = tensor("use_roll_5")]; + tensor var_312 = sub(x = var_17, y = use_roll_5)[name = tensor("op_312")]; + tensor var_313 = mul(x = filled_window_5, y = var_312)[name = tensor("op_313")]; + tensor var_314 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_314")]; + tensor window_7 = add(x = var_313, y = var_314)[name = tensor("window_7")]; + tensor var_316 = const()[name = tensor("op_316"), val = tensor(0x1p+0)]; + tensor var_317 = add(x = clip_3, y = var_316)[name = tensor("op_317")]; + tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_317)[name = tensor("clip_5")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_30, interleave = input_21_interleave_0, values = (window_3, window_5, window_7))[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; - tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; + tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; tensor inputs_3_pad_type_0 = const()[name = tensor("inputs_3_pad_type_0"), val = tensor("valid")]; tensor inputs_3_strides_0 = const()[name = tensor("inputs_3_strides_0"), val = tensor([1])]; tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_66")]; - tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_278_split_sizes_0 = const()[name = tensor("op_278_split_sizes_0"), val = tensor([256, 256])]; - tensor var_278_axis_0 = const()[name = tensor("op_278_axis_0"), val = tensor(1)]; - tensor var_278_0, tensor var_278_1 = split(axis = var_278_axis_0, split_sizes = var_278_split_sizes_0, x = inputs_3)[name = tensor("op_278")]; - tensor var_280 = sigmoid(x = var_278_1)[name = tensor("op_280")]; - tensor inputs_5 = mul(x = var_278_0, y = var_280)[name = tensor("inputs_5")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; + tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; + tensor var_340_split_sizes_0 = const()[name = tensor("op_340_split_sizes_0"), val = tensor([256, 256])]; + tensor var_340_axis_0 = const()[name = tensor("op_340_axis_0"), val = tensor(1)]; + tensor var_340_0, tensor var_340_1 = split(axis = var_340_axis_0, split_sizes = var_340_split_sizes_0, x = inputs_3)[name = tensor("op_340")]; + tensor var_342 = sigmoid(x = var_340_1)[name = tensor("op_342")]; + tensor inputs_5 = mul(x = var_340_0, y = var_342)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; tensor outputs_aug_1_strides_0 = const()[name = tensor("outputs_aug_1_strides_0"), val = tensor([1])]; tensor outputs_aug_1_dilations_0 = const()[name = tensor("outputs_aug_1_dilations_0"), val = tensor([1])]; - tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; + tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; tensor input_25_begin_0 = const()[name = tensor("input_25_begin_0"), val = tensor([0, 0, 0])]; - tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([1, 256, 16])]; + tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([3, 256, 16])]; tensor input_25_end_mask_0 = const()[name = tensor("input_25_end_mask_0"), val = tensor([true, true, false])]; - tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; - tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; - tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; + tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; + tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; + tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; tensor input_29_pad_type_0 = const()[name = tensor("input_29_pad_type_0"), val = tensor("valid")]; tensor input_29_strides_0 = const()[name = tensor("input_29_strides_0"), val = tensor([1])]; tensor input_29_pad_0 = const()[name = tensor("input_29_pad_0"), val = tensor([0, 0])]; tensor input_29_dilations_0 = const()[name = tensor("input_29_dilations_0"), val = tensor([1])]; tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; - tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; + tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_311_begin_0 = const()[name = tensor("op_311_begin_0"), val = tensor([0, -1, 0])]; - tensor var_311_end_0 = const()[name = tensor("op_311_end_0"), val = tensor([1, 16, 256])]; - tensor var_311_end_mask_0 = const()[name = tensor("op_311_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_65")]; - tensor var_311 = slice_by_index(begin = var_311_begin_0, end = var_311_end_0, end_mask = var_311_end_mask_0, x = conv_out_1)[name = tensor("op_311")]; - tensor var_314_begin_0 = const()[name = tensor("op_314_begin_0"), val = tensor([0, 1, 0])]; - tensor var_314_end_0 = const()[name = tensor("op_314_end_0"), val = tensor([1, 2, 256])]; - tensor var_314_end_mask_0 = const()[name = tensor("op_314_end_mask_0"), val = tensor([true, false, true])]; - tensor var_314 = slice_by_index(begin = var_314_begin_0, end = var_314_end_0, end_mask = var_314_end_mask_0, x = x_3)[name = tensor("op_314")]; - tensor var_317_begin_0 = const()[name = tensor("op_317_begin_0"), val = tensor([0, 1, 0])]; - tensor var_317_end_0 = const()[name = tensor("op_317_end_0"), val = tensor([1, 16, 256])]; - tensor var_317_end_mask_0 = const()[name = tensor("op_317_end_mask_0"), val = tensor([true, true, true])]; - tensor var_317 = slice_by_index(begin = var_317_begin_0, end = var_317_end_0, end_mask = var_317_end_mask_0, x = input_21)[name = tensor("op_317")]; - tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_317, var_314))[name = tensor("rolled_3")]; - tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; - tensor var_322 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_322")]; - tensor var_323_dtype_0 = const()[name = tensor("op_323_dtype_0"), val = tensor("fp32")]; - tensor var_324 = const()[name = tensor("op_324"), val = tensor([1, 16, 1])]; - tensor var_323 = cast(dtype = var_323_dtype_0, x = var_322)[name = tensor("cast_157")]; - tensor fill_mask_3 = reshape(shape = var_324, x = var_323)[name = tensor("fill_mask_3")]; - tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_314)[name = tensor("x_expanded_3")]; - tensor var_328 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_328")]; - tensor var_329 = mul(x = input_21, y = var_328)[name = tensor("op_329")]; - tensor var_330 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_330")]; - tensor filled_window_3 = add(x = var_329, y = var_330)[name = tensor("filled_window_3")]; - tensor var_332 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_332")]; - tensor var_333_dtype_0 = const()[name = tensor("op_333_dtype_0"), val = tensor("fp32")]; - tensor var_334 = const()[name = tensor("op_334"), val = tensor([1, 1, 1])]; - tensor var_333 = cast(dtype = var_333_dtype_0, x = var_332)[name = tensor("cast_156")]; - tensor use_roll_3 = reshape(shape = var_334, x = var_333)[name = tensor("use_roll_3")]; - tensor var_336 = sub(x = var_17, y = use_roll_3)[name = tensor("op_336")]; - tensor var_337 = mul(x = filled_window_3, y = var_336)[name = tensor("op_337")]; - tensor var_338 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_338")]; - tensor input_31 = add(x = var_337, y = var_338)[name = tensor("input_31")]; - tensor var_340 = const()[name = tensor("op_340"), val = tensor(0x1p+0)]; - tensor var_341 = add(x = clip_1, y = var_340)[name = tensor("op_341")]; - tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_341)[name = tensor("clip_3")]; + tensor var_373_begin_0 = const()[name = tensor("op_373_begin_0"), val = tensor([0, -1, 0])]; + tensor var_373_end_0 = const()[name = tensor("op_373_end_0"), val = tensor([3, 16, 256])]; + tensor var_373_end_mask_0 = const()[name = tensor("op_373_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_373 = slice_by_index(begin = var_373_begin_0, end = var_373_end_0, end_mask = var_373_end_mask_0, x = conv_out_1)[name = tensor("op_373")]; + tensor var_375_perm_0 = const()[name = tensor("op_375_perm_0"), val = tensor([1, 0, 2])]; + tensor var_375 = transpose(perm = var_375_perm_0, x = var_373)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_375)[name = tensor("input_31")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; + tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; + tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; + tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; + tensor var_398 = const()[name = tensor("op_398"), val = tensor(0x1p-1)]; + tensor var_399 = mul(x = input_39, y = var_398)[name = tensor("op_399")]; + tensor input_41 = add(x = var_399, y = input_31)[name = tensor("input_41")]; + tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; + tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; + tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; + tensor input_45 = layer_norm(axes = input_45_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_43)[name = tensor("input_45")]; + tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; + tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; + tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; + tensor var_428 = const()[name = tensor("op_428"), val = tensor(0x1p-1)]; + tensor var_429 = mul(x = input_51, y = var_428)[name = tensor("op_429")]; + tensor input_53 = add(x = var_429, y = input_43)[name = tensor("input_53")]; tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; - tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_31)[name = tensor("x_7")]; - tensor input_33_perm_0 = const()[name = tensor("input_33_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_9_pad_type_0 = const()[name = tensor("inputs_9_pad_type_0"), val = tensor("valid")]; - tensor inputs_9_strides_0 = const()[name = tensor("inputs_9_strides_0"), val = tensor([1])]; - tensor inputs_9_pad_0 = const()[name = tensor("inputs_9_pad_0"), val = tensor([0, 0])]; - tensor inputs_9_dilations_0 = const()[name = tensor("inputs_9_dilations_0"), val = tensor([1])]; - tensor inputs_9_groups_0 = const()[name = tensor("inputs_9_groups_0"), val = tensor(1)]; - tensor input_33 = transpose(perm = input_33_perm_0, x = x_7)[name = tensor("transpose_64")]; - tensor inputs_9 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_9_dilations_0, groups = inputs_9_groups_0, pad = inputs_9_pad_0, pad_type = inputs_9_pad_type_0, strides = inputs_9_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_33)[name = tensor("inputs_9")]; - tensor var_362_split_sizes_0 = const()[name = tensor("op_362_split_sizes_0"), val = tensor([256, 256])]; - tensor var_362_axis_0 = const()[name = tensor("op_362_axis_0"), val = tensor(1)]; - tensor var_362_0, tensor var_362_1 = split(axis = var_362_axis_0, split_sizes = var_362_split_sizes_0, x = inputs_9)[name = tensor("op_362")]; - tensor var_364 = sigmoid(x = var_362_1)[name = tensor("op_364")]; - tensor inputs_11 = mul(x = var_362_0, y = var_364)[name = tensor("inputs_11")]; - tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; - tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; - tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; - tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_11)[name = tensor("outputs_aug_3")]; - tensor input_35_begin_0 = const()[name = tensor("input_35_begin_0"), val = tensor([0, 0, 0])]; - tensor input_35_end_0 = const()[name = tensor("input_35_end_0"), val = tensor([1, 256, 16])]; - tensor input_35_end_mask_0 = const()[name = tensor("input_35_end_mask_0"), val = tensor([true, true, false])]; - tensor input_35 = slice_by_index(begin = input_35_begin_0, end = input_35_end_0, end_mask = input_35_end_mask_0, x = outputs_aug_3)[name = tensor("input_35")]; - tensor inputs_13 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_35)[name = tensor("inputs_13")]; - tensor input_37 = silu(x = inputs_13)[name = tensor("input_37")]; - tensor input_39_pad_type_0 = const()[name = tensor("input_39_pad_type_0"), val = tensor("valid")]; - tensor input_39_strides_0 = const()[name = tensor("input_39_strides_0"), val = tensor([1])]; - tensor input_39_pad_0 = const()[name = tensor("input_39_pad_0"), val = tensor([0, 0])]; - tensor input_39_dilations_0 = const()[name = tensor("input_39_dilations_0"), val = tensor([1])]; - tensor input_39_groups_0 = const()[name = tensor("input_39_groups_0"), val = tensor(1)]; - tensor input_39 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_37)[name = tensor("input_39")]; - tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_395_begin_0 = const()[name = tensor("op_395_begin_0"), val = tensor([0, -1, 0])]; - tensor var_395_end_0 = const()[name = tensor("op_395_end_0"), val = tensor([1, 16, 256])]; - tensor var_395_end_mask_0 = const()[name = tensor("op_395_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_39)[name = tensor("transpose_63")]; - tensor var_395 = slice_by_index(begin = var_395_begin_0, end = var_395_end_0, end_mask = var_395_end_mask_0, x = conv_out_3)[name = tensor("op_395")]; - tensor var_398_begin_0 = const()[name = tensor("op_398_begin_0"), val = tensor([0, 2, 0])]; - tensor var_398_end_0 = const()[name = tensor("op_398_end_0"), val = tensor([1, 1, 256])]; - tensor var_398_end_mask_0 = const()[name = tensor("op_398_end_mask_0"), val = tensor([true, true, true])]; - tensor var_398 = slice_by_index(begin = var_398_begin_0, end = var_398_end_0, end_mask = var_398_end_mask_0, x = x_3)[name = tensor("op_398")]; - tensor var_401_begin_0 = const()[name = tensor("op_401_begin_0"), val = tensor([0, 1, 0])]; - tensor var_401_end_0 = const()[name = tensor("op_401_end_0"), val = tensor([1, 16, 256])]; - tensor var_401_end_mask_0 = const()[name = tensor("op_401_end_mask_0"), val = tensor([true, true, true])]; - tensor var_401 = slice_by_index(begin = var_401_begin_0, end = var_401_end_0, end_mask = var_401_end_mask_0, x = input_31)[name = tensor("op_401")]; - tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_401, var_398))[name = tensor("rolled_5")]; - tensor clip_4 = clip(alpha = var_21, beta = var_20, x = clip_3)[name = tensor("clip_4")]; - tensor var_406 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_406")]; - tensor var_407_dtype_0 = const()[name = tensor("op_407_dtype_0"), val = tensor("fp32")]; - tensor var_408 = const()[name = tensor("op_408"), val = tensor([1, 16, 1])]; - tensor var_407 = cast(dtype = var_407_dtype_0, x = var_406)[name = tensor("cast_155")]; - tensor fill_mask_5 = reshape(shape = var_408, x = var_407)[name = tensor("fill_mask_5")]; - tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_398)[name = tensor("x_expanded_5")]; - tensor var_412 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_412")]; - tensor var_413 = mul(x = input_31, y = var_412)[name = tensor("op_413")]; - tensor var_414 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_414")]; - tensor filled_window_5 = add(x = var_413, y = var_414)[name = tensor("filled_window_5")]; - tensor var_416 = greater_equal(x = clip_3, y = var_16)[name = tensor("op_416")]; - tensor var_417_dtype_0 = const()[name = tensor("op_417_dtype_0"), val = tensor("fp32")]; - tensor var_418 = const()[name = tensor("op_418"), val = tensor([1, 1, 1])]; - tensor var_417 = cast(dtype = var_417_dtype_0, x = var_416)[name = tensor("cast_154")]; - tensor use_roll_5 = reshape(shape = var_418, x = var_417)[name = tensor("use_roll_5")]; - tensor var_420 = sub(x = var_17, y = use_roll_5)[name = tensor("op_420")]; - tensor var_421 = mul(x = filled_window_5, y = var_420)[name = tensor("op_421")]; - tensor var_422 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_422")]; - tensor input_41 = add(x = var_421, y = var_422)[name = tensor("input_41")]; - tensor var_424 = const()[name = tensor("op_424"), val = tensor(0x1p+0)]; - tensor var_425 = add(x = clip_3, y = var_424)[name = tensor("op_425")]; - tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_425)[name = tensor("clip_5")]; - tensor x_9_axes_0 = const()[name = tensor("x_9_axes_0"), val = tensor([-1])]; - tensor x_9 = layer_norm(axes = x_9_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_41)[name = tensor("x_9")]; - tensor input_43_perm_0 = const()[name = tensor("input_43_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_15_pad_type_0 = const()[name = tensor("inputs_15_pad_type_0"), val = tensor("valid")]; - tensor inputs_15_strides_0 = const()[name = tensor("inputs_15_strides_0"), val = tensor([1])]; - tensor inputs_15_pad_0 = const()[name = tensor("inputs_15_pad_0"), val = tensor([0, 0])]; - tensor inputs_15_dilations_0 = const()[name = tensor("inputs_15_dilations_0"), val = tensor([1])]; - tensor inputs_15_groups_0 = const()[name = tensor("inputs_15_groups_0"), val = tensor(1)]; - tensor input_43 = transpose(perm = input_43_perm_0, x = x_9)[name = tensor("transpose_62")]; - tensor inputs_15 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_15_dilations_0, groups = inputs_15_groups_0, pad = inputs_15_pad_0, pad_type = inputs_15_pad_type_0, strides = inputs_15_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_43)[name = tensor("inputs_15")]; - tensor var_446_split_sizes_0 = const()[name = tensor("op_446_split_sizes_0"), val = tensor([256, 256])]; - tensor var_446_axis_0 = const()[name = tensor("op_446_axis_0"), val = tensor(1)]; - tensor var_446_0, tensor var_446_1 = split(axis = var_446_axis_0, split_sizes = var_446_split_sizes_0, x = inputs_15)[name = tensor("op_446")]; - tensor var_448 = sigmoid(x = var_446_1)[name = tensor("op_448")]; - tensor inputs_17 = mul(x = var_446_0, y = var_448)[name = tensor("inputs_17")]; - tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; - tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; - tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; - tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_17)[name = tensor("outputs_aug_5")]; - tensor input_45_begin_0 = const()[name = tensor("input_45_begin_0"), val = tensor([0, 0, 0])]; - tensor input_45_end_0 = const()[name = tensor("input_45_end_0"), val = tensor([1, 256, 16])]; - tensor input_45_end_mask_0 = const()[name = tensor("input_45_end_mask_0"), val = tensor([true, true, false])]; - tensor input_45 = slice_by_index(begin = input_45_begin_0, end = input_45_end_0, end_mask = input_45_end_mask_0, x = outputs_aug_5)[name = tensor("input_45")]; - tensor inputs_19 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_45)[name = tensor("inputs_19")]; - tensor input_47 = silu(x = inputs_19)[name = tensor("input_47")]; - tensor input_49_pad_type_0 = const()[name = tensor("input_49_pad_type_0"), val = tensor("valid")]; - tensor input_49_strides_0 = const()[name = tensor("input_49_strides_0"), val = tensor([1])]; - tensor input_49_pad_0 = const()[name = tensor("input_49_pad_0"), val = tensor([0, 0])]; - tensor input_49_dilations_0 = const()[name = tensor("input_49_dilations_0"), val = tensor([1])]; - tensor input_49_groups_0 = const()[name = tensor("input_49_groups_0"), val = tensor(1)]; - tensor input_49 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_49_dilations_0, groups = input_49_groups_0, pad = input_49_pad_0, pad_type = input_49_pad_type_0, strides = input_49_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_47)[name = tensor("input_49")]; - tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_479_begin_0 = const()[name = tensor("op_479_begin_0"), val = tensor([0, -1, 0])]; - tensor var_479_end_0 = const()[name = tensor("op_479_end_0"), val = tensor([1, 16, 256])]; - tensor var_479_end_mask_0 = const()[name = tensor("op_479_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_49)[name = tensor("transpose_61")]; - tensor var_479 = slice_by_index(begin = var_479_begin_0, end = var_479_end_0, end_mask = var_479_end_mask_0, x = conv_out_5)[name = tensor("op_479")]; - tensor var_482_interleave_0 = const()[name = tensor("op_482_interleave_0"), val = tensor(false)]; - tensor var_482 = concat(axis = var_33, interleave = var_482_interleave_0, values = (var_311, var_395, var_479))[name = tensor("op_482")]; - tensor input_51 = add(x = x_3, y = var_482)[name = tensor("input_51")]; - tensor input_53_axes_0 = const()[name = tensor("input_53_axes_0"), val = tensor([-1])]; - tensor input_53 = layer_norm(axes = input_53_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_51)[name = tensor("input_53")]; - tensor inputs_21 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_53)[name = tensor("linear_8")]; - tensor input_55 = silu(x = inputs_21)[name = tensor("input_55")]; - tensor input_59 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_55)[name = tensor("linear_9")]; - tensor var_505 = const()[name = tensor("op_505"), val = tensor(0x1p-1)]; - tensor var_506 = mul(x = input_59, y = var_505)[name = tensor("op_506")]; - tensor input_61 = add(x = var_506, y = input_51)[name = tensor("input_61")]; - tensor input_63_axes_0 = const()[name = tensor("input_63_axes_0"), val = tensor([-1])]; - tensor input_63 = layer_norm(axes = input_63_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_61)[name = tensor("input_63")]; - tensor input_65_axes_0 = const()[name = tensor("input_65_axes_0"), val = tensor([-1])]; - tensor input_65 = layer_norm(axes = input_65_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_63)[name = tensor("input_65")]; - tensor inputs_23 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_65)[name = tensor("linear_10")]; - tensor input_67 = silu(x = inputs_23)[name = tensor("input_67")]; - tensor input_71 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_67)[name = tensor("linear_11")]; - tensor var_535 = const()[name = tensor("op_535"), val = tensor(0x1p-1)]; - tensor var_536 = mul(x = input_71, y = var_535)[name = tensor("op_536")]; - tensor input_73 = add(x = var_536, y = input_63)[name = tensor("input_73")]; - tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; - tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_73)[name = tensor("x_11")]; + tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_3_end_0 = const()[name = tensor("prev_kv_3_end_0"), val = tensor([2, 1, 4, 64, 64])]; tensor prev_kv_3_end_mask_0 = const()[name = tensor("prev_kv_3_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -563,316 +488,240 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_550 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_11)[name = tensor("linear_12")]; - tensor var_551 = const()[name = tensor("op_551"), val = tensor([1, 3, 4, 64])]; - tensor var_552 = reshape(shape = var_551, x = var_550)[name = tensor("op_552")]; + tensor var_443 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_444 = const()[name = tensor("op_444"), val = tensor([1, 3, 4, 64])]; + tensor var_445 = reshape(shape = var_444, x = var_443)[name = tensor("op_445")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_556 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_11)[name = tensor("linear_13")]; - tensor var_557 = const()[name = tensor("op_557"), val = tensor(0x1p-3)]; - tensor var_558 = mul(x = var_556, y = var_557)[name = tensor("op_558")]; - tensor var_559 = const()[name = tensor("op_559"), val = tensor([1, 3, 4, 64])]; - tensor var_560 = reshape(shape = var_559, x = var_558)[name = tensor("op_560")]; + tensor var_449 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_450 = const()[name = tensor("op_450"), val = tensor(0x1p-3)]; + tensor var_451 = mul(x = var_449, y = var_450)[name = tensor("op_451")]; + tensor var_452 = const()[name = tensor("op_452"), val = tensor([1, 3, 4, 64])]; + tensor var_453 = reshape(shape = var_452, x = var_451)[name = tensor("op_453")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_564 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_11)[name = tensor("linear_14")]; - tensor var_565 = const()[name = tensor("op_565"), val = tensor([1, 3, 4, 64])]; - tensor var_566 = reshape(shape = var_565, x = var_564)[name = tensor("op_566")]; + tensor var_457 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_458 = const()[name = tensor("op_458"), val = tensor([1, 3, 4, 64])]; + tensor var_459 = reshape(shape = var_458, x = var_457)[name = tensor("op_459")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_77 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_11)[name = tensor("linear_15")]; + tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; tensor s_t_3 = add(x = prev_scale_3, y = encoder__t_index)[name = tensor("s_t_3")]; tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_560)[name = tensor("transpose_59")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_552)[name = tensor("transpose_60")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_453)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_445)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_576 = const()[name = tensor("op_576"), val = tensor([3, 1])]; - tensor var_577 = reshape(shape = var_576, x = sqrt_s_t_3)[name = tensor("op_577")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_577)[name = tensor("M_3")]; - tensor var_579 = mul(x = qk_3, y = M_3)[name = tensor("op_579")]; + tensor var_469 = const()[name = tensor("op_469"), val = tensor([3, 1])]; + tensor var_470 = reshape(shape = var_469, x = sqrt_s_t_3)[name = tensor("op_470")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_470)[name = tensor("M_3")]; + tensor var_472 = mul(x = qk_3, y = M_3)[name = tensor("op_472")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_566)[name = tensor("transpose_58")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_579, y = v_3)[name = tensor("inner_3")]; - tensor var_581_transpose_x_0 = const()[name = tensor("op_581_transpose_x_0"), val = tensor(false)]; - tensor var_581_transpose_y_0 = const()[name = tensor("op_581_transpose_y_0"), val = tensor(false)]; - tensor var_581 = matmul(transpose_x = var_581_transpose_x_0, transpose_y = var_581_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_581")]; - tensor var_582 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_582")]; - tensor var_583 = const()[name = tensor("op_583"), val = tensor([1, 1, 3, 1])]; - tensor var_584 = reshape(shape = var_583, x = var_582)[name = tensor("op_584")]; - tensor cross_3 = mul(x = var_581, y = var_584)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_459)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_472, y = v_3)[name = tensor("inner_3")]; + tensor var_474_transpose_x_0 = const()[name = tensor("op_474_transpose_x_0"), val = tensor(false)]; + tensor var_474_transpose_y_0 = const()[name = tensor("op_474_transpose_y_0"), val = tensor(false)]; + tensor var_474 = matmul(transpose_x = var_474_transpose_x_0, transpose_y = var_474_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_474")]; + tensor var_475 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_475")]; + tensor var_476 = const()[name = tensor("op_476"), val = tensor([1, 1, 3, 1])]; + tensor var_477 = reshape(shape = var_476, x = var_475)[name = tensor("op_477")]; + tensor cross_3 = mul(x = var_474, y = var_477)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_587 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_587")]; - tensor var_589_transpose_x_1 = const()[name = tensor("op_589_transpose_x_1"), val = tensor(true)]; - tensor var_589_transpose_y_1 = const()[name = tensor("op_589_transpose_y_1"), val = tensor(false)]; - tensor var_589 = matmul(transpose_x = var_589_transpose_x_1, transpose_y = var_589_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_589")]; - tensor new_kv_unnorm_3 = add(x = var_587, y = var_589)[name = tensor("new_kv_unnorm_3")]; - tensor var_591 = const()[name = tensor("op_591"), val = tensor(0x1.8p+1)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_591)[name = tensor("new_scale_3")]; - tensor var_593 = sqrt(x = new_scale_3)[name = tensor("op_593")]; - tensor var_594 = real_div(x = new_kv_unnorm_3, y = var_593)[name = tensor("op_594")]; - tensor var_595_perm_0 = const()[name = tensor("op_595_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_480 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_480")]; + tensor var_482_transpose_x_1 = const()[name = tensor("op_482_transpose_x_1"), val = tensor(true)]; + tensor var_482_transpose_y_1 = const()[name = tensor("op_482_transpose_y_1"), val = tensor(false)]; + tensor var_482 = matmul(transpose_x = var_482_transpose_x_1, transpose_y = var_482_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_482")]; + tensor new_kv_unnorm_3 = add(x = var_480, y = var_482)[name = tensor("new_kv_unnorm_3")]; + tensor var_484 = const()[name = tensor("op_484"), val = tensor(0x1.8p+1)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_484)[name = tensor("new_scale_3")]; + tensor var_486 = sqrt(x = new_scale_3)[name = tensor("op_486")]; + tensor var_487 = real_div(x = new_kv_unnorm_3, y = var_486)[name = tensor("op_487")]; + tensor var_488_perm_0 = const()[name = tensor("op_488_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_595 = transpose(perm = var_595_perm_0, x = out_7)[name = tensor("transpose_57")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_595)[name = tensor("out_9")]; - tensor var_599 = const()[name = tensor("op_599"), val = tensor([1, 3, 256])]; - tensor out_11 = reshape(shape = var_599, x = out_9)[name = tensor("out_11")]; - tensor var_601 = silu(x = input_77)[name = tensor("op_601")]; - tensor input_79 = mul(x = var_601, y = out_11)[name = tensor("input_79")]; - tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_79)[name = tensor("linear_16")]; - tensor x_13 = add(x = input_73, y = ret_out_3)[name = tensor("x_13")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor var_488 = transpose(perm = var_488_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_488)[name = tensor("out_9")]; + tensor var_492 = const()[name = tensor("op_492"), val = tensor([1, 3, 256])]; + tensor out_11 = reshape(shape = var_492, x = out_9)[name = tensor("out_11")]; + tensor var_494 = silu(x = input_57)[name = tensor("op_494")]; + tensor input_59 = mul(x = var_494, y = out_11)[name = tensor("input_59")]; + tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; + tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; + tensor window_9_begin_0 = const()[name = tensor("window_9_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_9_end_0 = const()[name = tensor("window_9_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_9_end_mask_0 = const()[name = tensor("window_9_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_9_squeeze_mask_0 = const()[name = tensor("window_9_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_9 = slice_by_index(begin = window_9_begin_0, end = window_9_end_0, end_mask = window_9_end_mask_0, squeeze_mask = window_9_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_9")]; tensor filled_7_begin_0 = const()[name = tensor("filled_7_begin_0"), val = tensor([1, 0])]; tensor filled_7_end_0 = const()[name = tensor("filled_7_end_0"), val = tensor([2, 1])]; tensor filled_7_end_mask_0 = const()[name = tensor("filled_7_end_mask_0"), val = tensor([false, true])]; tensor filled_7_squeeze_mask_0 = const()[name = tensor("filled_7_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_7 = slice_by_index(begin = filled_7_begin_0, end = filled_7_end_0, end_mask = filled_7_end_mask_0, squeeze_mask = filled_7_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_7")]; - tensor var_610_begin_0 = const()[name = tensor("op_610_begin_0"), val = tensor([0, 0, 0])]; - tensor var_610_end_0 = const()[name = tensor("op_610_end_0"), val = tensor([1, 1, 256])]; - tensor var_610_end_mask_0 = const()[name = tensor("op_610_end_mask_0"), val = tensor([true, false, true])]; - tensor var_610 = slice_by_index(begin = var_610_begin_0, end = var_610_end_0, end_mask = var_610_end_mask_0, x = x_13)[name = tensor("op_610")]; - tensor var_613_begin_0 = const()[name = tensor("op_613_begin_0"), val = tensor([0, 1, 0])]; - tensor var_613_end_0 = const()[name = tensor("op_613_end_0"), val = tensor([1, 16, 256])]; - tensor var_613_end_mask_0 = const()[name = tensor("op_613_end_mask_0"), val = tensor([true, true, true])]; - tensor var_613 = slice_by_index(begin = var_613_begin_0, end = var_613_end_0, end_mask = var_613_end_mask_0, x = window_3)[name = tensor("op_613")]; + tensor var_503_begin_0 = const()[name = tensor("op_503_begin_0"), val = tensor([0, 0, 0])]; + tensor var_503_end_0 = const()[name = tensor("op_503_end_0"), val = tensor([1, 1, 256])]; + tensor var_503_end_mask_0 = const()[name = tensor("op_503_end_mask_0"), val = tensor([true, false, true])]; + tensor var_503 = slice_by_index(begin = var_503_begin_0, end = var_503_end_0, end_mask = var_503_end_mask_0, x = x_9)[name = tensor("op_503")]; + tensor var_506_begin_0 = const()[name = tensor("op_506_begin_0"), val = tensor([0, 1, 0])]; + tensor var_506_end_0 = const()[name = tensor("op_506_end_0"), val = tensor([1, 16, 256])]; + tensor var_506_end_mask_0 = const()[name = tensor("op_506_end_mask_0"), val = tensor([true, true, true])]; + tensor var_506 = slice_by_index(begin = var_506_begin_0, end = var_506_end_0, end_mask = var_506_end_mask_0, x = window_9)[name = tensor("op_506")]; tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; - tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_613, var_610))[name = tensor("rolled_7")]; + tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_506, var_503))[name = tensor("rolled_7")]; tensor clip_6 = clip(alpha = var_21, beta = var_20, x = filled_7)[name = tensor("clip_6")]; - tensor var_618 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_618")]; - tensor var_619_dtype_0 = const()[name = tensor("op_619_dtype_0"), val = tensor("fp32")]; - tensor var_620 = const()[name = tensor("op_620"), val = tensor([1, 16, 1])]; - tensor var_619 = cast(dtype = var_619_dtype_0, x = var_618)[name = tensor("cast_153")]; - tensor fill_mask_7 = reshape(shape = var_620, x = var_619)[name = tensor("fill_mask_7")]; + tensor var_511 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_511")]; + tensor var_512_dtype_0 = const()[name = tensor("op_512_dtype_0"), val = tensor("fp32")]; + tensor var_513 = const()[name = tensor("op_513"), val = tensor([1, 16, 1])]; + tensor var_512 = cast(dtype = var_512_dtype_0, x = var_511)[name = tensor("cast_153")]; + tensor fill_mask_7 = reshape(shape = var_513, x = var_512)[name = tensor("fill_mask_7")]; tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_610)[name = tensor("x_expanded_7")]; - tensor var_624 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_624")]; - tensor var_625 = mul(x = window_3, y = var_624)[name = tensor("op_625")]; - tensor var_626 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_626")]; - tensor filled_window_7 = add(x = var_625, y = var_626)[name = tensor("filled_window_7")]; - tensor var_628 = greater_equal(x = filled_7, y = var_16)[name = tensor("op_628")]; - tensor var_629_dtype_0 = const()[name = tensor("op_629_dtype_0"), val = tensor("fp32")]; - tensor var_630 = const()[name = tensor("op_630"), val = tensor([1, 1, 1])]; - tensor var_629 = cast(dtype = var_629_dtype_0, x = var_628)[name = tensor("cast_152")]; - tensor use_roll_7 = reshape(shape = var_630, x = var_629)[name = tensor("use_roll_7")]; - tensor var_632 = sub(x = var_17, y = use_roll_7)[name = tensor("op_632")]; - tensor var_633 = mul(x = filled_window_7, y = var_632)[name = tensor("op_633")]; - tensor var_634 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_634")]; - tensor input_81 = add(x = var_633, y = var_634)[name = tensor("input_81")]; - tensor var_636 = const()[name = tensor("op_636"), val = tensor(0x1p+0)]; - tensor var_637 = add(x = filled_7, y = var_636)[name = tensor("op_637")]; - tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_637)[name = tensor("clip_7")]; - tensor x_15_axes_0 = const()[name = tensor("x_15_axes_0"), val = tensor([-1])]; - tensor x_15 = layer_norm(axes = x_15_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_81)[name = tensor("x_15")]; - tensor input_83_perm_0 = const()[name = tensor("input_83_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_25_pad_type_0 = const()[name = tensor("inputs_25_pad_type_0"), val = tensor("valid")]; - tensor inputs_25_strides_0 = const()[name = tensor("inputs_25_strides_0"), val = tensor([1])]; - tensor inputs_25_pad_0 = const()[name = tensor("inputs_25_pad_0"), val = tensor([0, 0])]; - tensor inputs_25_dilations_0 = const()[name = tensor("inputs_25_dilations_0"), val = tensor([1])]; - tensor inputs_25_groups_0 = const()[name = tensor("inputs_25_groups_0"), val = tensor(1)]; - tensor input_83 = transpose(perm = input_83_perm_0, x = x_15)[name = tensor("transpose_56")]; - tensor inputs_25 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_25_dilations_0, groups = inputs_25_groups_0, pad = inputs_25_pad_0, pad_type = inputs_25_pad_type_0, strides = inputs_25_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_83)[name = tensor("inputs_25")]; - tensor var_658_split_sizes_0 = const()[name = tensor("op_658_split_sizes_0"), val = tensor([256, 256])]; - tensor var_658_axis_0 = const()[name = tensor("op_658_axis_0"), val = tensor(1)]; - tensor var_658_0, tensor var_658_1 = split(axis = var_658_axis_0, split_sizes = var_658_split_sizes_0, x = inputs_25)[name = tensor("op_658")]; - tensor var_660 = sigmoid(x = var_658_1)[name = tensor("op_660")]; - tensor inputs_27 = mul(x = var_658_0, y = var_660)[name = tensor("inputs_27")]; - tensor outputs_aug_7_pad_type_0 = const()[name = tensor("outputs_aug_7_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_7_pad_0 = const()[name = tensor("outputs_aug_7_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_7_groups_0 = const()[name = tensor("outputs_aug_7_groups_0"), val = tensor(256)]; - tensor outputs_aug_7_strides_0 = const()[name = tensor("outputs_aug_7_strides_0"), val = tensor([1])]; - tensor outputs_aug_7_dilations_0 = const()[name = tensor("outputs_aug_7_dilations_0"), val = tensor([1])]; - tensor outputs_aug_7 = conv(dilations = outputs_aug_7_dilations_0, groups = outputs_aug_7_groups_0, pad = outputs_aug_7_pad_0, pad_type = outputs_aug_7_pad_type_0, strides = outputs_aug_7_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_27)[name = tensor("outputs_aug_7")]; - tensor input_85_begin_0 = const()[name = tensor("input_85_begin_0"), val = tensor([0, 0, 0])]; - tensor input_85_end_0 = const()[name = tensor("input_85_end_0"), val = tensor([1, 256, 16])]; - tensor input_85_end_mask_0 = const()[name = tensor("input_85_end_mask_0"), val = tensor([true, true, false])]; - tensor input_85 = slice_by_index(begin = input_85_begin_0, end = input_85_end_0, end_mask = input_85_end_mask_0, x = outputs_aug_7)[name = tensor("input_85")]; - tensor inputs_29 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_85)[name = tensor("inputs_29")]; - tensor input_87 = silu(x = inputs_29)[name = tensor("input_87")]; - tensor input_89_pad_type_0 = const()[name = tensor("input_89_pad_type_0"), val = tensor("valid")]; - tensor input_89_strides_0 = const()[name = tensor("input_89_strides_0"), val = tensor([1])]; - tensor input_89_pad_0 = const()[name = tensor("input_89_pad_0"), val = tensor([0, 0])]; - tensor input_89_dilations_0 = const()[name = tensor("input_89_dilations_0"), val = tensor([1])]; - tensor input_89_groups_0 = const()[name = tensor("input_89_groups_0"), val = tensor(1)]; - tensor input_89 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_89_dilations_0, groups = input_89_groups_0, pad = input_89_pad_0, pad_type = input_89_pad_type_0, strides = input_89_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_87)[name = tensor("input_89")]; - tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_691_begin_0 = const()[name = tensor("op_691_begin_0"), val = tensor([0, -1, 0])]; - tensor var_691_end_0 = const()[name = tensor("op_691_end_0"), val = tensor([1, 16, 256])]; - tensor var_691_end_mask_0 = const()[name = tensor("op_691_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_89)[name = tensor("transpose_55")]; - tensor var_691 = slice_by_index(begin = var_691_begin_0, end = var_691_end_0, end_mask = var_691_end_mask_0, x = conv_out_7)[name = tensor("op_691")]; - tensor var_694_begin_0 = const()[name = tensor("op_694_begin_0"), val = tensor([0, 1, 0])]; - tensor var_694_end_0 = const()[name = tensor("op_694_end_0"), val = tensor([1, 2, 256])]; - tensor var_694_end_mask_0 = const()[name = tensor("op_694_end_mask_0"), val = tensor([true, false, true])]; - tensor var_694 = slice_by_index(begin = var_694_begin_0, end = var_694_end_0, end_mask = var_694_end_mask_0, x = x_13)[name = tensor("op_694")]; - tensor var_697_begin_0 = const()[name = tensor("op_697_begin_0"), val = tensor([0, 1, 0])]; - tensor var_697_end_0 = const()[name = tensor("op_697_end_0"), val = tensor([1, 16, 256])]; - tensor var_697_end_mask_0 = const()[name = tensor("op_697_end_mask_0"), val = tensor([true, true, true])]; - tensor var_697 = slice_by_index(begin = var_697_begin_0, end = var_697_end_0, end_mask = var_697_end_mask_0, x = input_81)[name = tensor("op_697")]; + tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_503)[name = tensor("x_expanded_7")]; + tensor var_517 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_517")]; + tensor var_518 = mul(x = window_9, y = var_517)[name = tensor("op_518")]; + tensor var_519 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_519")]; + tensor filled_window_7 = add(x = var_518, y = var_519)[name = tensor("filled_window_7")]; + tensor var_521 = greater_equal(x = filled_7, y = var_16)[name = tensor("op_521")]; + tensor var_522_dtype_0 = const()[name = tensor("op_522_dtype_0"), val = tensor("fp32")]; + tensor var_523 = const()[name = tensor("op_523"), val = tensor([1, 1, 1])]; + tensor var_522 = cast(dtype = var_522_dtype_0, x = var_521)[name = tensor("cast_152")]; + tensor use_roll_7 = reshape(shape = var_523, x = var_522)[name = tensor("use_roll_7")]; + tensor var_525 = sub(x = var_17, y = use_roll_7)[name = tensor("op_525")]; + tensor var_526 = mul(x = filled_window_7, y = var_525)[name = tensor("op_526")]; + tensor var_527 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_527")]; + tensor window_11 = add(x = var_526, y = var_527)[name = tensor("window_11")]; + tensor var_529 = const()[name = tensor("op_529"), val = tensor(0x1p+0)]; + tensor var_530 = add(x = filled_7, y = var_529)[name = tensor("op_530")]; + tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_530)[name = tensor("clip_7")]; + tensor var_533_begin_0 = const()[name = tensor("op_533_begin_0"), val = tensor([0, 1, 0])]; + tensor var_533_end_0 = const()[name = tensor("op_533_end_0"), val = tensor([1, 2, 256])]; + tensor var_533_end_mask_0 = const()[name = tensor("op_533_end_mask_0"), val = tensor([true, false, true])]; + tensor var_533 = slice_by_index(begin = var_533_begin_0, end = var_533_end_0, end_mask = var_533_end_mask_0, x = x_9)[name = tensor("op_533")]; + tensor var_536_begin_0 = const()[name = tensor("op_536_begin_0"), val = tensor([0, 1, 0])]; + tensor var_536_end_0 = const()[name = tensor("op_536_end_0"), val = tensor([1, 16, 256])]; + tensor var_536_end_mask_0 = const()[name = tensor("op_536_end_mask_0"), val = tensor([true, true, true])]; + tensor var_536 = slice_by_index(begin = var_536_begin_0, end = var_536_end_0, end_mask = var_536_end_mask_0, x = window_11)[name = tensor("op_536")]; tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; - tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_697, var_694))[name = tensor("rolled_9")]; + tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_536, var_533))[name = tensor("rolled_9")]; tensor clip_8 = clip(alpha = var_21, beta = var_20, x = clip_7)[name = tensor("clip_8")]; - tensor var_702 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_702")]; - tensor var_703_dtype_0 = const()[name = tensor("op_703_dtype_0"), val = tensor("fp32")]; - tensor var_704 = const()[name = tensor("op_704"), val = tensor([1, 16, 1])]; - tensor var_703 = cast(dtype = var_703_dtype_0, x = var_702)[name = tensor("cast_151")]; - tensor fill_mask_9 = reshape(shape = var_704, x = var_703)[name = tensor("fill_mask_9")]; + tensor var_541 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_541")]; + tensor var_542_dtype_0 = const()[name = tensor("op_542_dtype_0"), val = tensor("fp32")]; + tensor var_543 = const()[name = tensor("op_543"), val = tensor([1, 16, 1])]; + tensor var_542 = cast(dtype = var_542_dtype_0, x = var_541)[name = tensor("cast_151")]; + tensor fill_mask_9 = reshape(shape = var_543, x = var_542)[name = tensor("fill_mask_9")]; tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_694)[name = tensor("x_expanded_9")]; - tensor var_708 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_708")]; - tensor var_709 = mul(x = input_81, y = var_708)[name = tensor("op_709")]; - tensor var_710 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_710")]; - tensor filled_window_9 = add(x = var_709, y = var_710)[name = tensor("filled_window_9")]; - tensor var_712 = greater_equal(x = clip_7, y = var_16)[name = tensor("op_712")]; - tensor var_713_dtype_0 = const()[name = tensor("op_713_dtype_0"), val = tensor("fp32")]; - tensor var_714 = const()[name = tensor("op_714"), val = tensor([1, 1, 1])]; - tensor var_713 = cast(dtype = var_713_dtype_0, x = var_712)[name = tensor("cast_150")]; - tensor use_roll_9 = reshape(shape = var_714, x = var_713)[name = tensor("use_roll_9")]; - tensor var_716 = sub(x = var_17, y = use_roll_9)[name = tensor("op_716")]; - tensor var_717 = mul(x = filled_window_9, y = var_716)[name = tensor("op_717")]; - tensor var_718 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_718")]; - tensor input_91 = add(x = var_717, y = var_718)[name = tensor("input_91")]; - tensor var_720 = const()[name = tensor("op_720"), val = tensor(0x1p+0)]; - tensor var_721 = add(x = clip_7, y = var_720)[name = tensor("op_721")]; - tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_721)[name = tensor("clip_9")]; - tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; - tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_91)[name = tensor("x_17")]; - tensor input_93_perm_0 = const()[name = tensor("input_93_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_31_pad_type_0 = const()[name = tensor("inputs_31_pad_type_0"), val = tensor("valid")]; - tensor inputs_31_strides_0 = const()[name = tensor("inputs_31_strides_0"), val = tensor([1])]; - tensor inputs_31_pad_0 = const()[name = tensor("inputs_31_pad_0"), val = tensor([0, 0])]; - tensor inputs_31_dilations_0 = const()[name = tensor("inputs_31_dilations_0"), val = tensor([1])]; - tensor inputs_31_groups_0 = const()[name = tensor("inputs_31_groups_0"), val = tensor(1)]; - tensor input_93 = transpose(perm = input_93_perm_0, x = x_17)[name = tensor("transpose_54")]; - tensor inputs_31 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_31_dilations_0, groups = inputs_31_groups_0, pad = inputs_31_pad_0, pad_type = inputs_31_pad_type_0, strides = inputs_31_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_93)[name = tensor("inputs_31")]; - tensor var_742_split_sizes_0 = const()[name = tensor("op_742_split_sizes_0"), val = tensor([256, 256])]; - tensor var_742_axis_0 = const()[name = tensor("op_742_axis_0"), val = tensor(1)]; - tensor var_742_0, tensor var_742_1 = split(axis = var_742_axis_0, split_sizes = var_742_split_sizes_0, x = inputs_31)[name = tensor("op_742")]; - tensor var_744 = sigmoid(x = var_742_1)[name = tensor("op_744")]; - tensor inputs_33 = mul(x = var_742_0, y = var_744)[name = tensor("inputs_33")]; - tensor outputs_aug_9_pad_type_0 = const()[name = tensor("outputs_aug_9_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_9_pad_0 = const()[name = tensor("outputs_aug_9_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_9_groups_0 = const()[name = tensor("outputs_aug_9_groups_0"), val = tensor(256)]; - tensor outputs_aug_9_strides_0 = const()[name = tensor("outputs_aug_9_strides_0"), val = tensor([1])]; - tensor outputs_aug_9_dilations_0 = const()[name = tensor("outputs_aug_9_dilations_0"), val = tensor([1])]; - tensor outputs_aug_9 = conv(dilations = outputs_aug_9_dilations_0, groups = outputs_aug_9_groups_0, pad = outputs_aug_9_pad_0, pad_type = outputs_aug_9_pad_type_0, strides = outputs_aug_9_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_33)[name = tensor("outputs_aug_9")]; - tensor input_95_begin_0 = const()[name = tensor("input_95_begin_0"), val = tensor([0, 0, 0])]; - tensor input_95_end_0 = const()[name = tensor("input_95_end_0"), val = tensor([1, 256, 16])]; - tensor input_95_end_mask_0 = const()[name = tensor("input_95_end_mask_0"), val = tensor([true, true, false])]; - tensor input_95 = slice_by_index(begin = input_95_begin_0, end = input_95_end_0, end_mask = input_95_end_mask_0, x = outputs_aug_9)[name = tensor("input_95")]; - tensor inputs_35 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_95)[name = tensor("inputs_35")]; - tensor input_97 = silu(x = inputs_35)[name = tensor("input_97")]; - tensor input_99_pad_type_0 = const()[name = tensor("input_99_pad_type_0"), val = tensor("valid")]; - tensor input_99_strides_0 = const()[name = tensor("input_99_strides_0"), val = tensor([1])]; - tensor input_99_pad_0 = const()[name = tensor("input_99_pad_0"), val = tensor([0, 0])]; - tensor input_99_dilations_0 = const()[name = tensor("input_99_dilations_0"), val = tensor([1])]; - tensor input_99_groups_0 = const()[name = tensor("input_99_groups_0"), val = tensor(1)]; - tensor input_99 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_99_dilations_0, groups = input_99_groups_0, pad = input_99_pad_0, pad_type = input_99_pad_type_0, strides = input_99_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_97)[name = tensor("input_99")]; - tensor conv_out_9_perm_0 = const()[name = tensor("conv_out_9_perm_0"), val = tensor([0, 2, 1])]; - tensor var_775_begin_0 = const()[name = tensor("op_775_begin_0"), val = tensor([0, -1, 0])]; - tensor var_775_end_0 = const()[name = tensor("op_775_end_0"), val = tensor([1, 16, 256])]; - tensor var_775_end_mask_0 = const()[name = tensor("op_775_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_9 = transpose(perm = conv_out_9_perm_0, x = input_99)[name = tensor("transpose_53")]; - tensor var_775 = slice_by_index(begin = var_775_begin_0, end = var_775_end_0, end_mask = var_775_end_mask_0, x = conv_out_9)[name = tensor("op_775")]; - tensor var_778_begin_0 = const()[name = tensor("op_778_begin_0"), val = tensor([0, 2, 0])]; - tensor var_778_end_0 = const()[name = tensor("op_778_end_0"), val = tensor([1, 1, 256])]; - tensor var_778_end_mask_0 = const()[name = tensor("op_778_end_mask_0"), val = tensor([true, true, true])]; - tensor var_778 = slice_by_index(begin = var_778_begin_0, end = var_778_end_0, end_mask = var_778_end_mask_0, x = x_13)[name = tensor("op_778")]; - tensor var_781_begin_0 = const()[name = tensor("op_781_begin_0"), val = tensor([0, 1, 0])]; - tensor var_781_end_0 = const()[name = tensor("op_781_end_0"), val = tensor([1, 16, 256])]; - tensor var_781_end_mask_0 = const()[name = tensor("op_781_end_mask_0"), val = tensor([true, true, true])]; - tensor var_781 = slice_by_index(begin = var_781_begin_0, end = var_781_end_0, end_mask = var_781_end_mask_0, x = input_91)[name = tensor("op_781")]; + tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_533)[name = tensor("x_expanded_9")]; + tensor var_547 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_547")]; + tensor var_548 = mul(x = window_11, y = var_547)[name = tensor("op_548")]; + tensor var_549 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_549")]; + tensor filled_window_9 = add(x = var_548, y = var_549)[name = tensor("filled_window_9")]; + tensor var_551 = greater_equal(x = clip_7, y = var_16)[name = tensor("op_551")]; + tensor var_552_dtype_0 = const()[name = tensor("op_552_dtype_0"), val = tensor("fp32")]; + tensor var_553 = const()[name = tensor("op_553"), val = tensor([1, 1, 1])]; + tensor var_552 = cast(dtype = var_552_dtype_0, x = var_551)[name = tensor("cast_150")]; + tensor use_roll_9 = reshape(shape = var_553, x = var_552)[name = tensor("use_roll_9")]; + tensor var_555 = sub(x = var_17, y = use_roll_9)[name = tensor("op_555")]; + tensor var_556 = mul(x = filled_window_9, y = var_555)[name = tensor("op_556")]; + tensor var_557 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_557")]; + tensor window_13 = add(x = var_556, y = var_557)[name = tensor("window_13")]; + tensor var_559 = const()[name = tensor("op_559"), val = tensor(0x1p+0)]; + tensor var_560 = add(x = clip_7, y = var_559)[name = tensor("op_560")]; + tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_560)[name = tensor("clip_9")]; + tensor var_563_begin_0 = const()[name = tensor("op_563_begin_0"), val = tensor([0, 2, 0])]; + tensor var_563_end_0 = const()[name = tensor("op_563_end_0"), val = tensor([1, 1, 256])]; + tensor var_563_end_mask_0 = const()[name = tensor("op_563_end_mask_0"), val = tensor([true, true, true])]; + tensor var_563 = slice_by_index(begin = var_563_begin_0, end = var_563_end_0, end_mask = var_563_end_mask_0, x = x_9)[name = tensor("op_563")]; + tensor var_566_begin_0 = const()[name = tensor("op_566_begin_0"), val = tensor([0, 1, 0])]; + tensor var_566_end_0 = const()[name = tensor("op_566_end_0"), val = tensor([1, 16, 256])]; + tensor var_566_end_mask_0 = const()[name = tensor("op_566_end_mask_0"), val = tensor([true, true, true])]; + tensor var_566 = slice_by_index(begin = var_566_begin_0, end = var_566_end_0, end_mask = var_566_end_mask_0, x = window_13)[name = tensor("op_566")]; tensor rolled_11_interleave_0 = const()[name = tensor("rolled_11_interleave_0"), val = tensor(false)]; - tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_781, var_778))[name = tensor("rolled_11")]; + tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_566, var_563))[name = tensor("rolled_11")]; tensor clip_10 = clip(alpha = var_21, beta = var_20, x = clip_9)[name = tensor("clip_10")]; - tensor var_786 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_786")]; - tensor var_787_dtype_0 = const()[name = tensor("op_787_dtype_0"), val = tensor("fp32")]; - tensor var_788 = const()[name = tensor("op_788"), val = tensor([1, 16, 1])]; - tensor var_787 = cast(dtype = var_787_dtype_0, x = var_786)[name = tensor("cast_149")]; - tensor fill_mask_11 = reshape(shape = var_788, x = var_787)[name = tensor("fill_mask_11")]; + tensor var_571 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_571")]; + tensor var_572_dtype_0 = const()[name = tensor("op_572_dtype_0"), val = tensor("fp32")]; + tensor var_573 = const()[name = tensor("op_573"), val = tensor([1, 16, 1])]; + tensor var_572 = cast(dtype = var_572_dtype_0, x = var_571)[name = tensor("cast_149")]; + tensor fill_mask_11 = reshape(shape = var_573, x = var_572)[name = tensor("fill_mask_11")]; tensor x_expanded_11_reps_0 = const()[name = tensor("x_expanded_11_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_778)[name = tensor("x_expanded_11")]; - tensor var_792 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_792")]; - tensor var_793 = mul(x = input_91, y = var_792)[name = tensor("op_793")]; - tensor var_794 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_794")]; - tensor filled_window_11 = add(x = var_793, y = var_794)[name = tensor("filled_window_11")]; - tensor var_796 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_796")]; - tensor var_797_dtype_0 = const()[name = tensor("op_797_dtype_0"), val = tensor("fp32")]; - tensor var_798 = const()[name = tensor("op_798"), val = tensor([1, 1, 1])]; - tensor var_797 = cast(dtype = var_797_dtype_0, x = var_796)[name = tensor("cast_148")]; - tensor use_roll_11 = reshape(shape = var_798, x = var_797)[name = tensor("use_roll_11")]; - tensor var_800 = sub(x = var_17, y = use_roll_11)[name = tensor("op_800")]; - tensor var_801 = mul(x = filled_window_11, y = var_800)[name = tensor("op_801")]; - tensor var_802 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_802")]; - tensor input_101 = add(x = var_801, y = var_802)[name = tensor("input_101")]; - tensor var_804 = const()[name = tensor("op_804"), val = tensor(0x1p+0)]; - tensor var_805 = add(x = clip_9, y = var_804)[name = tensor("op_805")]; - tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_805)[name = tensor("clip_11")]; - tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; - tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_101)[name = tensor("x_19")]; - tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_37_pad_type_0 = const()[name = tensor("inputs_37_pad_type_0"), val = tensor("valid")]; - tensor inputs_37_strides_0 = const()[name = tensor("inputs_37_strides_0"), val = tensor([1])]; - tensor inputs_37_pad_0 = const()[name = tensor("inputs_37_pad_0"), val = tensor([0, 0])]; - tensor inputs_37_dilations_0 = const()[name = tensor("inputs_37_dilations_0"), val = tensor([1])]; - tensor inputs_37_groups_0 = const()[name = tensor("inputs_37_groups_0"), val = tensor(1)]; - tensor input_103 = transpose(perm = input_103_perm_0, x = x_19)[name = tensor("transpose_52")]; - tensor inputs_37 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_37_dilations_0, groups = inputs_37_groups_0, pad = inputs_37_pad_0, pad_type = inputs_37_pad_type_0, strides = inputs_37_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_37")]; - tensor var_826_split_sizes_0 = const()[name = tensor("op_826_split_sizes_0"), val = tensor([256, 256])]; - tensor var_826_axis_0 = const()[name = tensor("op_826_axis_0"), val = tensor(1)]; - tensor var_826_0, tensor var_826_1 = split(axis = var_826_axis_0, split_sizes = var_826_split_sizes_0, x = inputs_37)[name = tensor("op_826")]; - tensor var_828 = sigmoid(x = var_826_1)[name = tensor("op_828")]; - tensor inputs_39 = mul(x = var_826_0, y = var_828)[name = tensor("inputs_39")]; - tensor outputs_aug_11_pad_type_0 = const()[name = tensor("outputs_aug_11_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_11_pad_0 = const()[name = tensor("outputs_aug_11_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_11_groups_0 = const()[name = tensor("outputs_aug_11_groups_0"), val = tensor(256)]; - tensor outputs_aug_11_strides_0 = const()[name = tensor("outputs_aug_11_strides_0"), val = tensor([1])]; - tensor outputs_aug_11_dilations_0 = const()[name = tensor("outputs_aug_11_dilations_0"), val = tensor([1])]; - tensor outputs_aug_11 = conv(dilations = outputs_aug_11_dilations_0, groups = outputs_aug_11_groups_0, pad = outputs_aug_11_pad_0, pad_type = outputs_aug_11_pad_type_0, strides = outputs_aug_11_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_39)[name = tensor("outputs_aug_11")]; - tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; - tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([1, 256, 16])]; - tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; - tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_11)[name = tensor("input_105")]; - tensor inputs_41 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_105)[name = tensor("inputs_41")]; - tensor input_107 = silu(x = inputs_41)[name = tensor("input_107")]; - tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; - tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; - tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; - tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; - tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; - tensor input_109 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; - tensor conv_out_11_perm_0 = const()[name = tensor("conv_out_11_perm_0"), val = tensor([0, 2, 1])]; - tensor var_859_begin_0 = const()[name = tensor("op_859_begin_0"), val = tensor([0, -1, 0])]; - tensor var_859_end_0 = const()[name = tensor("op_859_end_0"), val = tensor([1, 16, 256])]; - tensor var_859_end_mask_0 = const()[name = tensor("op_859_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_11 = transpose(perm = conv_out_11_perm_0, x = input_109)[name = tensor("transpose_51")]; - tensor var_859 = slice_by_index(begin = var_859_begin_0, end = var_859_end_0, end_mask = var_859_end_mask_0, x = conv_out_11)[name = tensor("op_859")]; - tensor var_862_interleave_0 = const()[name = tensor("op_862_interleave_0"), val = tensor(false)]; - tensor var_862 = concat(axis = var_33, interleave = var_862_interleave_0, values = (var_691, var_775, var_859))[name = tensor("op_862")]; - tensor input_111 = add(x = x_13, y = var_862)[name = tensor("input_111")]; - tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; - tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; - tensor inputs_43 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_17")]; - tensor input_115 = silu(x = inputs_43)[name = tensor("input_115")]; - tensor input_119 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_18")]; - tensor var_885 = const()[name = tensor("op_885"), val = tensor(0x1p-1)]; - tensor var_886 = mul(x = input_119, y = var_885)[name = tensor("op_886")]; - tensor input_121 = add(x = var_886, y = input_111)[name = tensor("input_121")]; - tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; - tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_121)[name = tensor("input_123")]; - tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; - tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; - tensor inputs_45 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_19")]; - tensor input_127 = silu(x = inputs_45)[name = tensor("input_127")]; - tensor input_131 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_20")]; - tensor var_915 = const()[name = tensor("op_915"), val = tensor(0x1p-1)]; - tensor var_916 = mul(x = input_131, y = var_915)[name = tensor("op_916")]; - tensor input_133 = add(x = var_916, y = input_123)[name = tensor("input_133")]; - tensor x_21_axes_0 = const()[name = tensor("x_21_axes_0"), val = tensor([-1])]; - tensor x_21 = layer_norm(axes = x_21_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_133)[name = tensor("x_21")]; + tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_563)[name = tensor("x_expanded_11")]; + tensor var_577 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_577")]; + tensor var_578 = mul(x = window_13, y = var_577)[name = tensor("op_578")]; + tensor var_579 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_579")]; + tensor filled_window_11 = add(x = var_578, y = var_579)[name = tensor("filled_window_11")]; + tensor var_581 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_581")]; + tensor var_582_dtype_0 = const()[name = tensor("op_582_dtype_0"), val = tensor("fp32")]; + tensor var_583 = const()[name = tensor("op_583"), val = tensor([1, 1, 1])]; + tensor var_582 = cast(dtype = var_582_dtype_0, x = var_581)[name = tensor("cast_148")]; + tensor use_roll_11 = reshape(shape = var_583, x = var_582)[name = tensor("use_roll_11")]; + tensor var_585 = sub(x = var_17, y = use_roll_11)[name = tensor("op_585")]; + tensor var_586 = mul(x = filled_window_11, y = var_585)[name = tensor("op_586")]; + tensor var_587 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_587")]; + tensor window_15 = add(x = var_586, y = var_587)[name = tensor("window_15")]; + tensor var_589 = const()[name = tensor("op_589"), val = tensor(0x1p+0)]; + tensor var_590 = add(x = clip_9, y = var_589)[name = tensor("op_590")]; + tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_590)[name = tensor("clip_11")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_30, interleave = input_61_interleave_0, values = (window_11, window_13, window_15))[name = tensor("input_61")]; + tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; + tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; + tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_13_pad_type_0 = const()[name = tensor("inputs_13_pad_type_0"), val = tensor("valid")]; + tensor inputs_13_strides_0 = const()[name = tensor("inputs_13_strides_0"), val = tensor([1])]; + tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; + tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; + tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; + tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; + tensor var_613_split_sizes_0 = const()[name = tensor("op_613_split_sizes_0"), val = tensor([256, 256])]; + tensor var_613_axis_0 = const()[name = tensor("op_613_axis_0"), val = tensor(1)]; + tensor var_613_0, tensor var_613_1 = split(axis = var_613_axis_0, split_sizes = var_613_split_sizes_0, x = inputs_13)[name = tensor("op_613")]; + tensor var_615 = sigmoid(x = var_613_1)[name = tensor("op_615")]; + tensor inputs_15 = mul(x = var_613_0, y = var_615)[name = tensor("inputs_15")]; + tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; + tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; + tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; + tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_15)[name = tensor("outputs_aug_3")]; + tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; + tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([3, 256, 16])]; + tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; + tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_3)[name = tensor("input_65")]; + tensor inputs_17 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_65)[name = tensor("inputs_17")]; + tensor input_67 = silu(x = inputs_17)[name = tensor("input_67")]; + tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; + tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; + tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; + tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; + tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; + tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; + tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; + tensor var_646_begin_0 = const()[name = tensor("op_646_begin_0"), val = tensor([0, -1, 0])]; + tensor var_646_end_0 = const()[name = tensor("op_646_end_0"), val = tensor([3, 16, 256])]; + tensor var_646_end_mask_0 = const()[name = tensor("op_646_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_646 = slice_by_index(begin = var_646_begin_0, end = var_646_end_0, end_mask = var_646_end_mask_0, x = conv_out_3)[name = tensor("op_646")]; + tensor var_648_perm_0 = const()[name = tensor("op_648_perm_0"), val = tensor([1, 0, 2])]; + tensor var_648 = transpose(perm = var_648_perm_0, x = var_646)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_648)[name = tensor("input_71")]; + tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; + tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; + tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; + tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; + tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; + tensor var_671 = const()[name = tensor("op_671"), val = tensor(0x1p-1)]; + tensor var_672 = mul(x = input_79, y = var_671)[name = tensor("op_672")]; + tensor input_81 = add(x = var_672, y = input_71)[name = tensor("input_81")]; + tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; + tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; + tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; + tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; + tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; + tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; + tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; + tensor var_701 = const()[name = tensor("op_701"), val = tensor(0x1p-1)]; + tensor var_702 = mul(x = input_91, y = var_701)[name = tensor("op_702")]; + tensor input_93 = add(x = var_702, y = input_83)[name = tensor("input_93")]; + tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; + tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; tensor prev_kv_5_end_0 = const()[name = tensor("prev_kv_5_end_0"), val = tensor([3, 1, 4, 64, 64])]; tensor prev_kv_5_end_mask_0 = const()[name = tensor("prev_kv_5_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -883,316 +732,240 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_930 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_21)[name = tensor("linear_21")]; - tensor var_931 = const()[name = tensor("op_931"), val = tensor([1, 3, 4, 64])]; - tensor var_932 = reshape(shape = var_931, x = var_930)[name = tensor("op_932")]; + tensor var_716 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_717 = const()[name = tensor("op_717"), val = tensor([1, 3, 4, 64])]; + tensor var_718 = reshape(shape = var_717, x = var_716)[name = tensor("op_718")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_936 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_21)[name = tensor("linear_22")]; - tensor var_937 = const()[name = tensor("op_937"), val = tensor(0x1p-3)]; - tensor var_938 = mul(x = var_936, y = var_937)[name = tensor("op_938")]; - tensor var_939 = const()[name = tensor("op_939"), val = tensor([1, 3, 4, 64])]; - tensor var_940 = reshape(shape = var_939, x = var_938)[name = tensor("op_940")]; + tensor var_722 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_723 = const()[name = tensor("op_723"), val = tensor(0x1p-3)]; + tensor var_724 = mul(x = var_722, y = var_723)[name = tensor("op_724")]; + tensor var_725 = const()[name = tensor("op_725"), val = tensor([1, 3, 4, 64])]; + tensor var_726 = reshape(shape = var_725, x = var_724)[name = tensor("op_726")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_944 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_21)[name = tensor("linear_23")]; - tensor var_945 = const()[name = tensor("op_945"), val = tensor([1, 3, 4, 64])]; - tensor var_946 = reshape(shape = var_945, x = var_944)[name = tensor("op_946")]; + tensor var_730 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_731 = const()[name = tensor("op_731"), val = tensor([1, 3, 4, 64])]; + tensor var_732 = reshape(shape = var_731, x = var_730)[name = tensor("op_732")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_137 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_21)[name = tensor("linear_24")]; + tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; tensor s_t_5 = add(x = prev_scale_5, y = encoder__t_index)[name = tensor("s_t_5")]; tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_940)[name = tensor("transpose_49")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_932)[name = tensor("transpose_50")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_726)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_718)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_956 = const()[name = tensor("op_956"), val = tensor([3, 1])]; - tensor var_957 = reshape(shape = var_956, x = sqrt_s_t_5)[name = tensor("op_957")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_957)[name = tensor("M_5")]; - tensor var_959 = mul(x = qk_5, y = M_5)[name = tensor("op_959")]; + tensor var_742 = const()[name = tensor("op_742"), val = tensor([3, 1])]; + tensor var_743 = reshape(shape = var_742, x = sqrt_s_t_5)[name = tensor("op_743")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_743)[name = tensor("M_5")]; + tensor var_745 = mul(x = qk_5, y = M_5)[name = tensor("op_745")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_946)[name = tensor("transpose_48")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_959, y = v_5)[name = tensor("inner_5")]; - tensor var_961_transpose_x_0 = const()[name = tensor("op_961_transpose_x_0"), val = tensor(false)]; - tensor var_961_transpose_y_0 = const()[name = tensor("op_961_transpose_y_0"), val = tensor(false)]; - tensor var_961 = matmul(transpose_x = var_961_transpose_x_0, transpose_y = var_961_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_961")]; - tensor var_962 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_962")]; - tensor var_963 = const()[name = tensor("op_963"), val = tensor([1, 1, 3, 1])]; - tensor var_964 = reshape(shape = var_963, x = var_962)[name = tensor("op_964")]; - tensor cross_5 = mul(x = var_961, y = var_964)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_732)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_745, y = v_5)[name = tensor("inner_5")]; + tensor var_747_transpose_x_0 = const()[name = tensor("op_747_transpose_x_0"), val = tensor(false)]; + tensor var_747_transpose_y_0 = const()[name = tensor("op_747_transpose_y_0"), val = tensor(false)]; + tensor var_747 = matmul(transpose_x = var_747_transpose_x_0, transpose_y = var_747_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_747")]; + tensor var_748 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_748")]; + tensor var_749 = const()[name = tensor("op_749"), val = tensor([1, 1, 3, 1])]; + tensor var_750 = reshape(shape = var_749, x = var_748)[name = tensor("op_750")]; + tensor cross_5 = mul(x = var_747, y = var_750)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_967 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_967")]; - tensor var_969_transpose_x_1 = const()[name = tensor("op_969_transpose_x_1"), val = tensor(true)]; - tensor var_969_transpose_y_1 = const()[name = tensor("op_969_transpose_y_1"), val = tensor(false)]; - tensor var_969 = matmul(transpose_x = var_969_transpose_x_1, transpose_y = var_969_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_969")]; - tensor new_kv_unnorm_5 = add(x = var_967, y = var_969)[name = tensor("new_kv_unnorm_5")]; - tensor var_971 = const()[name = tensor("op_971"), val = tensor(0x1.8p+1)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_971)[name = tensor("new_scale_5")]; - tensor var_973 = sqrt(x = new_scale_5)[name = tensor("op_973")]; - tensor var_974 = real_div(x = new_kv_unnorm_5, y = var_973)[name = tensor("op_974")]; - tensor var_975_perm_0 = const()[name = tensor("op_975_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_753 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_753")]; + tensor var_755_transpose_x_1 = const()[name = tensor("op_755_transpose_x_1"), val = tensor(true)]; + tensor var_755_transpose_y_1 = const()[name = tensor("op_755_transpose_y_1"), val = tensor(false)]; + tensor var_755 = matmul(transpose_x = var_755_transpose_x_1, transpose_y = var_755_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_755")]; + tensor new_kv_unnorm_5 = add(x = var_753, y = var_755)[name = tensor("new_kv_unnorm_5")]; + tensor var_757 = const()[name = tensor("op_757"), val = tensor(0x1.8p+1)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_757)[name = tensor("new_scale_5")]; + tensor var_759 = sqrt(x = new_scale_5)[name = tensor("op_759")]; + tensor var_760 = real_div(x = new_kv_unnorm_5, y = var_759)[name = tensor("op_760")]; + tensor var_761_perm_0 = const()[name = tensor("op_761_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_975 = transpose(perm = var_975_perm_0, x = out_13)[name = tensor("transpose_47")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_975)[name = tensor("out_15")]; - tensor var_979 = const()[name = tensor("op_979"), val = tensor([1, 3, 256])]; - tensor out_17 = reshape(shape = var_979, x = out_15)[name = tensor("out_17")]; - tensor var_981 = silu(x = input_137)[name = tensor("op_981")]; - tensor input_139 = mul(x = var_981, y = out_17)[name = tensor("input_139")]; - tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_139)[name = tensor("linear_25")]; - tensor x_23 = add(x = input_133, y = ret_out_5)[name = tensor("x_23")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor var_761 = transpose(perm = var_761_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_761)[name = tensor("out_15")]; + tensor var_765 = const()[name = tensor("op_765"), val = tensor([1, 3, 256])]; + tensor out_17 = reshape(shape = var_765, x = out_15)[name = tensor("out_17")]; + tensor var_767 = silu(x = input_97)[name = tensor("op_767")]; + tensor input_99 = mul(x = var_767, y = out_17)[name = tensor("input_99")]; + tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; + tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; + tensor window_17_begin_0 = const()[name = tensor("window_17_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_17_end_0 = const()[name = tensor("window_17_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_17_end_mask_0 = const()[name = tensor("window_17_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_17_squeeze_mask_0 = const()[name = tensor("window_17_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_17 = slice_by_index(begin = window_17_begin_0, end = window_17_end_0, end_mask = window_17_end_mask_0, squeeze_mask = window_17_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_17")]; tensor filled_13_begin_0 = const()[name = tensor("filled_13_begin_0"), val = tensor([2, 0])]; tensor filled_13_end_0 = const()[name = tensor("filled_13_end_0"), val = tensor([3, 1])]; tensor filled_13_end_mask_0 = const()[name = tensor("filled_13_end_mask_0"), val = tensor([false, true])]; tensor filled_13_squeeze_mask_0 = const()[name = tensor("filled_13_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_13 = slice_by_index(begin = filled_13_begin_0, end = filled_13_end_0, end_mask = filled_13_end_mask_0, squeeze_mask = filled_13_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_13")]; - tensor var_990_begin_0 = const()[name = tensor("op_990_begin_0"), val = tensor([0, 0, 0])]; - tensor var_990_end_0 = const()[name = tensor("op_990_end_0"), val = tensor([1, 1, 256])]; - tensor var_990_end_mask_0 = const()[name = tensor("op_990_end_mask_0"), val = tensor([true, false, true])]; - tensor var_990 = slice_by_index(begin = var_990_begin_0, end = var_990_end_0, end_mask = var_990_end_mask_0, x = x_23)[name = tensor("op_990")]; - tensor var_993_begin_0 = const()[name = tensor("op_993_begin_0"), val = tensor([0, 1, 0])]; - tensor var_993_end_0 = const()[name = tensor("op_993_end_0"), val = tensor([1, 16, 256])]; - tensor var_993_end_mask_0 = const()[name = tensor("op_993_end_mask_0"), val = tensor([true, true, true])]; - tensor var_993 = slice_by_index(begin = var_993_begin_0, end = var_993_end_0, end_mask = var_993_end_mask_0, x = window_5)[name = tensor("op_993")]; + tensor var_776_begin_0 = const()[name = tensor("op_776_begin_0"), val = tensor([0, 0, 0])]; + tensor var_776_end_0 = const()[name = tensor("op_776_end_0"), val = tensor([1, 1, 256])]; + tensor var_776_end_mask_0 = const()[name = tensor("op_776_end_mask_0"), val = tensor([true, false, true])]; + tensor var_776 = slice_by_index(begin = var_776_begin_0, end = var_776_end_0, end_mask = var_776_end_mask_0, x = x_15)[name = tensor("op_776")]; + tensor var_779_begin_0 = const()[name = tensor("op_779_begin_0"), val = tensor([0, 1, 0])]; + tensor var_779_end_0 = const()[name = tensor("op_779_end_0"), val = tensor([1, 16, 256])]; + tensor var_779_end_mask_0 = const()[name = tensor("op_779_end_mask_0"), val = tensor([true, true, true])]; + tensor var_779 = slice_by_index(begin = var_779_begin_0, end = var_779_end_0, end_mask = var_779_end_mask_0, x = window_17)[name = tensor("op_779")]; tensor rolled_13_interleave_0 = const()[name = tensor("rolled_13_interleave_0"), val = tensor(false)]; - tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_993, var_990))[name = tensor("rolled_13")]; + tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_779, var_776))[name = tensor("rolled_13")]; tensor clip_12 = clip(alpha = var_21, beta = var_20, x = filled_13)[name = tensor("clip_12")]; - tensor var_998 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_998")]; - tensor var_999_dtype_0 = const()[name = tensor("op_999_dtype_0"), val = tensor("fp32")]; - tensor var_1000 = const()[name = tensor("op_1000"), val = tensor([1, 16, 1])]; - tensor var_999 = cast(dtype = var_999_dtype_0, x = var_998)[name = tensor("cast_147")]; - tensor fill_mask_13 = reshape(shape = var_1000, x = var_999)[name = tensor("fill_mask_13")]; + tensor var_784 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_784")]; + tensor var_785_dtype_0 = const()[name = tensor("op_785_dtype_0"), val = tensor("fp32")]; + tensor var_786 = const()[name = tensor("op_786"), val = tensor([1, 16, 1])]; + tensor var_785 = cast(dtype = var_785_dtype_0, x = var_784)[name = tensor("cast_147")]; + tensor fill_mask_13 = reshape(shape = var_786, x = var_785)[name = tensor("fill_mask_13")]; tensor x_expanded_13_reps_0 = const()[name = tensor("x_expanded_13_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_990)[name = tensor("x_expanded_13")]; - tensor var_1004 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_1004")]; - tensor var_1005 = mul(x = window_5, y = var_1004)[name = tensor("op_1005")]; - tensor var_1006 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_1006")]; - tensor filled_window_13 = add(x = var_1005, y = var_1006)[name = tensor("filled_window_13")]; - tensor var_1008 = greater_equal(x = filled_13, y = var_16)[name = tensor("op_1008")]; - tensor var_1009_dtype_0 = const()[name = tensor("op_1009_dtype_0"), val = tensor("fp32")]; - tensor var_1010 = const()[name = tensor("op_1010"), val = tensor([1, 1, 1])]; - tensor var_1009 = cast(dtype = var_1009_dtype_0, x = var_1008)[name = tensor("cast_146")]; - tensor use_roll_13 = reshape(shape = var_1010, x = var_1009)[name = tensor("use_roll_13")]; - tensor var_1012 = sub(x = var_17, y = use_roll_13)[name = tensor("op_1012")]; - tensor var_1013 = mul(x = filled_window_13, y = var_1012)[name = tensor("op_1013")]; - tensor var_1014 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_1014")]; - tensor input_141 = add(x = var_1013, y = var_1014)[name = tensor("input_141")]; - tensor var_1016 = const()[name = tensor("op_1016"), val = tensor(0x1p+0)]; - tensor var_1017 = add(x = filled_13, y = var_1016)[name = tensor("op_1017")]; - tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_1017)[name = tensor("clip_13")]; - tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; - tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_141)[name = tensor("x_25")]; - tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_47_pad_type_0 = const()[name = tensor("inputs_47_pad_type_0"), val = tensor("valid")]; - tensor inputs_47_strides_0 = const()[name = tensor("inputs_47_strides_0"), val = tensor([1])]; - tensor inputs_47_pad_0 = const()[name = tensor("inputs_47_pad_0"), val = tensor([0, 0])]; - tensor inputs_47_dilations_0 = const()[name = tensor("inputs_47_dilations_0"), val = tensor([1])]; - tensor inputs_47_groups_0 = const()[name = tensor("inputs_47_groups_0"), val = tensor(1)]; - tensor input_143 = transpose(perm = input_143_perm_0, x = x_25)[name = tensor("transpose_46")]; - tensor inputs_47 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_47_dilations_0, groups = inputs_47_groups_0, pad = inputs_47_pad_0, pad_type = inputs_47_pad_type_0, strides = inputs_47_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_47")]; - tensor var_1038_split_sizes_0 = const()[name = tensor("op_1038_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1038_axis_0 = const()[name = tensor("op_1038_axis_0"), val = tensor(1)]; - tensor var_1038_0, tensor var_1038_1 = split(axis = var_1038_axis_0, split_sizes = var_1038_split_sizes_0, x = inputs_47)[name = tensor("op_1038")]; - tensor var_1040 = sigmoid(x = var_1038_1)[name = tensor("op_1040")]; - tensor inputs_49 = mul(x = var_1038_0, y = var_1040)[name = tensor("inputs_49")]; - tensor outputs_aug_13_pad_type_0 = const()[name = tensor("outputs_aug_13_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_13_pad_0 = const()[name = tensor("outputs_aug_13_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_13_groups_0 = const()[name = tensor("outputs_aug_13_groups_0"), val = tensor(256)]; - tensor outputs_aug_13_strides_0 = const()[name = tensor("outputs_aug_13_strides_0"), val = tensor([1])]; - tensor outputs_aug_13_dilations_0 = const()[name = tensor("outputs_aug_13_dilations_0"), val = tensor([1])]; - tensor outputs_aug_13 = conv(dilations = outputs_aug_13_dilations_0, groups = outputs_aug_13_groups_0, pad = outputs_aug_13_pad_0, pad_type = outputs_aug_13_pad_type_0, strides = outputs_aug_13_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_49)[name = tensor("outputs_aug_13")]; - tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; - tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([1, 256, 16])]; - tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; - tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug_13)[name = tensor("input_145")]; - tensor inputs_51 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_145)[name = tensor("inputs_51")]; - tensor input_147 = silu(x = inputs_51)[name = tensor("input_147")]; - tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; - tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; - tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; - tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; - tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; - tensor input_149 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; - tensor conv_out_13_perm_0 = const()[name = tensor("conv_out_13_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1071_begin_0 = const()[name = tensor("op_1071_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1071_end_0 = const()[name = tensor("op_1071_end_0"), val = tensor([1, 16, 256])]; - tensor var_1071_end_mask_0 = const()[name = tensor("op_1071_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_13 = transpose(perm = conv_out_13_perm_0, x = input_149)[name = tensor("transpose_45")]; - tensor var_1071 = slice_by_index(begin = var_1071_begin_0, end = var_1071_end_0, end_mask = var_1071_end_mask_0, x = conv_out_13)[name = tensor("op_1071")]; - tensor var_1074_begin_0 = const()[name = tensor("op_1074_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1074_end_0 = const()[name = tensor("op_1074_end_0"), val = tensor([1, 2, 256])]; - tensor var_1074_end_mask_0 = const()[name = tensor("op_1074_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1074 = slice_by_index(begin = var_1074_begin_0, end = var_1074_end_0, end_mask = var_1074_end_mask_0, x = x_23)[name = tensor("op_1074")]; - tensor var_1077_begin_0 = const()[name = tensor("op_1077_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1077_end_0 = const()[name = tensor("op_1077_end_0"), val = tensor([1, 16, 256])]; - tensor var_1077_end_mask_0 = const()[name = tensor("op_1077_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1077 = slice_by_index(begin = var_1077_begin_0, end = var_1077_end_0, end_mask = var_1077_end_mask_0, x = input_141)[name = tensor("op_1077")]; + tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_776)[name = tensor("x_expanded_13")]; + tensor var_790 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_790")]; + tensor var_791 = mul(x = window_17, y = var_790)[name = tensor("op_791")]; + tensor var_792 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_792")]; + tensor filled_window_13 = add(x = var_791, y = var_792)[name = tensor("filled_window_13")]; + tensor var_794 = greater_equal(x = filled_13, y = var_16)[name = tensor("op_794")]; + tensor var_795_dtype_0 = const()[name = tensor("op_795_dtype_0"), val = tensor("fp32")]; + tensor var_796 = const()[name = tensor("op_796"), val = tensor([1, 1, 1])]; + tensor var_795 = cast(dtype = var_795_dtype_0, x = var_794)[name = tensor("cast_146")]; + tensor use_roll_13 = reshape(shape = var_796, x = var_795)[name = tensor("use_roll_13")]; + tensor var_798 = sub(x = var_17, y = use_roll_13)[name = tensor("op_798")]; + tensor var_799 = mul(x = filled_window_13, y = var_798)[name = tensor("op_799")]; + tensor var_800 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_800")]; + tensor window_19 = add(x = var_799, y = var_800)[name = tensor("window_19")]; + tensor var_802 = const()[name = tensor("op_802"), val = tensor(0x1p+0)]; + tensor var_803 = add(x = filled_13, y = var_802)[name = tensor("op_803")]; + tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_803)[name = tensor("clip_13")]; + tensor var_806_begin_0 = const()[name = tensor("op_806_begin_0"), val = tensor([0, 1, 0])]; + tensor var_806_end_0 = const()[name = tensor("op_806_end_0"), val = tensor([1, 2, 256])]; + tensor var_806_end_mask_0 = const()[name = tensor("op_806_end_mask_0"), val = tensor([true, false, true])]; + tensor var_806 = slice_by_index(begin = var_806_begin_0, end = var_806_end_0, end_mask = var_806_end_mask_0, x = x_15)[name = tensor("op_806")]; + tensor var_809_begin_0 = const()[name = tensor("op_809_begin_0"), val = tensor([0, 1, 0])]; + tensor var_809_end_0 = const()[name = tensor("op_809_end_0"), val = tensor([1, 16, 256])]; + tensor var_809_end_mask_0 = const()[name = tensor("op_809_end_mask_0"), val = tensor([true, true, true])]; + tensor var_809 = slice_by_index(begin = var_809_begin_0, end = var_809_end_0, end_mask = var_809_end_mask_0, x = window_19)[name = tensor("op_809")]; tensor rolled_15_interleave_0 = const()[name = tensor("rolled_15_interleave_0"), val = tensor(false)]; - tensor rolled_15 = concat(axis = var_33, interleave = rolled_15_interleave_0, values = (var_1077, var_1074))[name = tensor("rolled_15")]; + tensor rolled_15 = concat(axis = var_33, interleave = rolled_15_interleave_0, values = (var_809, var_806))[name = tensor("rolled_15")]; tensor clip_14 = clip(alpha = var_21, beta = var_20, x = clip_13)[name = tensor("clip_14")]; - tensor var_1082 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_1082")]; - tensor var_1083_dtype_0 = const()[name = tensor("op_1083_dtype_0"), val = tensor("fp32")]; - tensor var_1084 = const()[name = tensor("op_1084"), val = tensor([1, 16, 1])]; - tensor var_1083 = cast(dtype = var_1083_dtype_0, x = var_1082)[name = tensor("cast_145")]; - tensor fill_mask_15 = reshape(shape = var_1084, x = var_1083)[name = tensor("fill_mask_15")]; + tensor var_814 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_814")]; + tensor var_815_dtype_0 = const()[name = tensor("op_815_dtype_0"), val = tensor("fp32")]; + tensor var_816 = const()[name = tensor("op_816"), val = tensor([1, 16, 1])]; + tensor var_815 = cast(dtype = var_815_dtype_0, x = var_814)[name = tensor("cast_145")]; + tensor fill_mask_15 = reshape(shape = var_816, x = var_815)[name = tensor("fill_mask_15")]; tensor x_expanded_15_reps_0 = const()[name = tensor("x_expanded_15_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_1074)[name = tensor("x_expanded_15")]; - tensor var_1088 = sub(x = var_17, y = fill_mask_15)[name = tensor("op_1088")]; - tensor var_1089 = mul(x = input_141, y = var_1088)[name = tensor("op_1089")]; - tensor var_1090 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_1090")]; - tensor filled_window_15 = add(x = var_1089, y = var_1090)[name = tensor("filled_window_15")]; - tensor var_1092 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_1092")]; - tensor var_1093_dtype_0 = const()[name = tensor("op_1093_dtype_0"), val = tensor("fp32")]; - tensor var_1094 = const()[name = tensor("op_1094"), val = tensor([1, 1, 1])]; - tensor var_1093 = cast(dtype = var_1093_dtype_0, x = var_1092)[name = tensor("cast_144")]; - tensor use_roll_15 = reshape(shape = var_1094, x = var_1093)[name = tensor("use_roll_15")]; - tensor var_1096 = sub(x = var_17, y = use_roll_15)[name = tensor("op_1096")]; - tensor var_1097 = mul(x = filled_window_15, y = var_1096)[name = tensor("op_1097")]; - tensor var_1098 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_1098")]; - tensor input_151 = add(x = var_1097, y = var_1098)[name = tensor("input_151")]; - tensor var_1100 = const()[name = tensor("op_1100"), val = tensor(0x1p+0)]; - tensor var_1101 = add(x = clip_13, y = var_1100)[name = tensor("op_1101")]; - tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_1101)[name = tensor("clip_15")]; - tensor x_27_axes_0 = const()[name = tensor("x_27_axes_0"), val = tensor([-1])]; - tensor x_27 = layer_norm(axes = x_27_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_151)[name = tensor("x_27")]; - tensor input_153_perm_0 = const()[name = tensor("input_153_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_53_pad_type_0 = const()[name = tensor("inputs_53_pad_type_0"), val = tensor("valid")]; - tensor inputs_53_strides_0 = const()[name = tensor("inputs_53_strides_0"), val = tensor([1])]; - tensor inputs_53_pad_0 = const()[name = tensor("inputs_53_pad_0"), val = tensor([0, 0])]; - tensor inputs_53_dilations_0 = const()[name = tensor("inputs_53_dilations_0"), val = tensor([1])]; - tensor inputs_53_groups_0 = const()[name = tensor("inputs_53_groups_0"), val = tensor(1)]; - tensor input_153 = transpose(perm = input_153_perm_0, x = x_27)[name = tensor("transpose_44")]; - tensor inputs_53 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_53_dilations_0, groups = inputs_53_groups_0, pad = inputs_53_pad_0, pad_type = inputs_53_pad_type_0, strides = inputs_53_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_153)[name = tensor("inputs_53")]; - tensor var_1122_split_sizes_0 = const()[name = tensor("op_1122_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1122_axis_0 = const()[name = tensor("op_1122_axis_0"), val = tensor(1)]; - tensor var_1122_0, tensor var_1122_1 = split(axis = var_1122_axis_0, split_sizes = var_1122_split_sizes_0, x = inputs_53)[name = tensor("op_1122")]; - tensor var_1124 = sigmoid(x = var_1122_1)[name = tensor("op_1124")]; - tensor inputs_55 = mul(x = var_1122_0, y = var_1124)[name = tensor("inputs_55")]; - tensor outputs_aug_15_pad_type_0 = const()[name = tensor("outputs_aug_15_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_15_pad_0 = const()[name = tensor("outputs_aug_15_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_15_groups_0 = const()[name = tensor("outputs_aug_15_groups_0"), val = tensor(256)]; - tensor outputs_aug_15_strides_0 = const()[name = tensor("outputs_aug_15_strides_0"), val = tensor([1])]; - tensor outputs_aug_15_dilations_0 = const()[name = tensor("outputs_aug_15_dilations_0"), val = tensor([1])]; - tensor outputs_aug_15 = conv(dilations = outputs_aug_15_dilations_0, groups = outputs_aug_15_groups_0, pad = outputs_aug_15_pad_0, pad_type = outputs_aug_15_pad_type_0, strides = outputs_aug_15_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_55)[name = tensor("outputs_aug_15")]; - tensor input_155_begin_0 = const()[name = tensor("input_155_begin_0"), val = tensor([0, 0, 0])]; - tensor input_155_end_0 = const()[name = tensor("input_155_end_0"), val = tensor([1, 256, 16])]; - tensor input_155_end_mask_0 = const()[name = tensor("input_155_end_mask_0"), val = tensor([true, true, false])]; - tensor input_155 = slice_by_index(begin = input_155_begin_0, end = input_155_end_0, end_mask = input_155_end_mask_0, x = outputs_aug_15)[name = tensor("input_155")]; - tensor inputs_57 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_155)[name = tensor("inputs_57")]; - tensor input_157 = silu(x = inputs_57)[name = tensor("input_157")]; - tensor input_159_pad_type_0 = const()[name = tensor("input_159_pad_type_0"), val = tensor("valid")]; - tensor input_159_strides_0 = const()[name = tensor("input_159_strides_0"), val = tensor([1])]; - tensor input_159_pad_0 = const()[name = tensor("input_159_pad_0"), val = tensor([0, 0])]; - tensor input_159_dilations_0 = const()[name = tensor("input_159_dilations_0"), val = tensor([1])]; - tensor input_159_groups_0 = const()[name = tensor("input_159_groups_0"), val = tensor(1)]; - tensor input_159 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_159_dilations_0, groups = input_159_groups_0, pad = input_159_pad_0, pad_type = input_159_pad_type_0, strides = input_159_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_157)[name = tensor("input_159")]; - tensor conv_out_15_perm_0 = const()[name = tensor("conv_out_15_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1155_begin_0 = const()[name = tensor("op_1155_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1155_end_0 = const()[name = tensor("op_1155_end_0"), val = tensor([1, 16, 256])]; - tensor var_1155_end_mask_0 = const()[name = tensor("op_1155_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_15 = transpose(perm = conv_out_15_perm_0, x = input_159)[name = tensor("transpose_43")]; - tensor var_1155 = slice_by_index(begin = var_1155_begin_0, end = var_1155_end_0, end_mask = var_1155_end_mask_0, x = conv_out_15)[name = tensor("op_1155")]; - tensor var_1158_begin_0 = const()[name = tensor("op_1158_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1158_end_0 = const()[name = tensor("op_1158_end_0"), val = tensor([1, 1, 256])]; - tensor var_1158_end_mask_0 = const()[name = tensor("op_1158_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1158 = slice_by_index(begin = var_1158_begin_0, end = var_1158_end_0, end_mask = var_1158_end_mask_0, x = x_23)[name = tensor("op_1158")]; - tensor var_1161_begin_0 = const()[name = tensor("op_1161_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1161_end_0 = const()[name = tensor("op_1161_end_0"), val = tensor([1, 16, 256])]; - tensor var_1161_end_mask_0 = const()[name = tensor("op_1161_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1161 = slice_by_index(begin = var_1161_begin_0, end = var_1161_end_0, end_mask = var_1161_end_mask_0, x = input_151)[name = tensor("op_1161")]; + tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_806)[name = tensor("x_expanded_15")]; + tensor var_820 = sub(x = var_17, y = fill_mask_15)[name = tensor("op_820")]; + tensor var_821 = mul(x = window_19, y = var_820)[name = tensor("op_821")]; + tensor var_822 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_822")]; + tensor filled_window_15 = add(x = var_821, y = var_822)[name = tensor("filled_window_15")]; + tensor var_824 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_824")]; + tensor var_825_dtype_0 = const()[name = tensor("op_825_dtype_0"), val = tensor("fp32")]; + tensor var_826 = const()[name = tensor("op_826"), val = tensor([1, 1, 1])]; + tensor var_825 = cast(dtype = var_825_dtype_0, x = var_824)[name = tensor("cast_144")]; + tensor use_roll_15 = reshape(shape = var_826, x = var_825)[name = tensor("use_roll_15")]; + tensor var_828 = sub(x = var_17, y = use_roll_15)[name = tensor("op_828")]; + tensor var_829 = mul(x = filled_window_15, y = var_828)[name = tensor("op_829")]; + tensor var_830 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_830")]; + tensor window_21 = add(x = var_829, y = var_830)[name = tensor("window_21")]; + tensor var_832 = const()[name = tensor("op_832"), val = tensor(0x1p+0)]; + tensor var_833 = add(x = clip_13, y = var_832)[name = tensor("op_833")]; + tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_833)[name = tensor("clip_15")]; + tensor var_836_begin_0 = const()[name = tensor("op_836_begin_0"), val = tensor([0, 2, 0])]; + tensor var_836_end_0 = const()[name = tensor("op_836_end_0"), val = tensor([1, 1, 256])]; + tensor var_836_end_mask_0 = const()[name = tensor("op_836_end_mask_0"), val = tensor([true, true, true])]; + tensor var_836 = slice_by_index(begin = var_836_begin_0, end = var_836_end_0, end_mask = var_836_end_mask_0, x = x_15)[name = tensor("op_836")]; + tensor var_839_begin_0 = const()[name = tensor("op_839_begin_0"), val = tensor([0, 1, 0])]; + tensor var_839_end_0 = const()[name = tensor("op_839_end_0"), val = tensor([1, 16, 256])]; + tensor var_839_end_mask_0 = const()[name = tensor("op_839_end_mask_0"), val = tensor([true, true, true])]; + tensor var_839 = slice_by_index(begin = var_839_begin_0, end = var_839_end_0, end_mask = var_839_end_mask_0, x = window_21)[name = tensor("op_839")]; tensor rolled_17_interleave_0 = const()[name = tensor("rolled_17_interleave_0"), val = tensor(false)]; - tensor rolled_17 = concat(axis = var_33, interleave = rolled_17_interleave_0, values = (var_1161, var_1158))[name = tensor("rolled_17")]; + tensor rolled_17 = concat(axis = var_33, interleave = rolled_17_interleave_0, values = (var_839, var_836))[name = tensor("rolled_17")]; tensor clip_16 = clip(alpha = var_21, beta = var_20, x = clip_15)[name = tensor("clip_16")]; - tensor var_1166 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_1166")]; - tensor var_1167_dtype_0 = const()[name = tensor("op_1167_dtype_0"), val = tensor("fp32")]; - tensor var_1168 = const()[name = tensor("op_1168"), val = tensor([1, 16, 1])]; - tensor var_1167 = cast(dtype = var_1167_dtype_0, x = var_1166)[name = tensor("cast_143")]; - tensor fill_mask_17 = reshape(shape = var_1168, x = var_1167)[name = tensor("fill_mask_17")]; + tensor var_844 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_844")]; + tensor var_845_dtype_0 = const()[name = tensor("op_845_dtype_0"), val = tensor("fp32")]; + tensor var_846 = const()[name = tensor("op_846"), val = tensor([1, 16, 1])]; + tensor var_845 = cast(dtype = var_845_dtype_0, x = var_844)[name = tensor("cast_143")]; + tensor fill_mask_17 = reshape(shape = var_846, x = var_845)[name = tensor("fill_mask_17")]; tensor x_expanded_17_reps_0 = const()[name = tensor("x_expanded_17_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_1158)[name = tensor("x_expanded_17")]; - tensor var_1172 = sub(x = var_17, y = fill_mask_17)[name = tensor("op_1172")]; - tensor var_1173 = mul(x = input_151, y = var_1172)[name = tensor("op_1173")]; - tensor var_1174 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_1174")]; - tensor filled_window_17 = add(x = var_1173, y = var_1174)[name = tensor("filled_window_17")]; - tensor var_1176 = greater_equal(x = clip_15, y = var_16)[name = tensor("op_1176")]; - tensor var_1177_dtype_0 = const()[name = tensor("op_1177_dtype_0"), val = tensor("fp32")]; - tensor var_1178 = const()[name = tensor("op_1178"), val = tensor([1, 1, 1])]; - tensor var_1177 = cast(dtype = var_1177_dtype_0, x = var_1176)[name = tensor("cast_142")]; - tensor use_roll_17 = reshape(shape = var_1178, x = var_1177)[name = tensor("use_roll_17")]; - tensor var_1180 = sub(x = var_17, y = use_roll_17)[name = tensor("op_1180")]; - tensor var_1181 = mul(x = filled_window_17, y = var_1180)[name = tensor("op_1181")]; - tensor var_1182 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_1182")]; - tensor input_161 = add(x = var_1181, y = var_1182)[name = tensor("input_161")]; - tensor var_1184 = const()[name = tensor("op_1184"), val = tensor(0x1p+0)]; - tensor var_1185 = add(x = clip_15, y = var_1184)[name = tensor("op_1185")]; - tensor clip_17 = clip(alpha = var_21, beta = var_16, x = var_1185)[name = tensor("clip_17")]; - tensor x_29_axes_0 = const()[name = tensor("x_29_axes_0"), val = tensor([-1])]; - tensor x_29 = layer_norm(axes = x_29_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_161)[name = tensor("x_29")]; - tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_59_pad_type_0 = const()[name = tensor("inputs_59_pad_type_0"), val = tensor("valid")]; - tensor inputs_59_strides_0 = const()[name = tensor("inputs_59_strides_0"), val = tensor([1])]; - tensor inputs_59_pad_0 = const()[name = tensor("inputs_59_pad_0"), val = tensor([0, 0])]; - tensor inputs_59_dilations_0 = const()[name = tensor("inputs_59_dilations_0"), val = tensor([1])]; - tensor inputs_59_groups_0 = const()[name = tensor("inputs_59_groups_0"), val = tensor(1)]; - tensor input_163 = transpose(perm = input_163_perm_0, x = x_29)[name = tensor("transpose_42")]; - tensor inputs_59 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_59_dilations_0, groups = inputs_59_groups_0, pad = inputs_59_pad_0, pad_type = inputs_59_pad_type_0, strides = inputs_59_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_163)[name = tensor("inputs_59")]; - tensor var_1206_split_sizes_0 = const()[name = tensor("op_1206_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1206_axis_0 = const()[name = tensor("op_1206_axis_0"), val = tensor(1)]; - tensor var_1206_0, tensor var_1206_1 = split(axis = var_1206_axis_0, split_sizes = var_1206_split_sizes_0, x = inputs_59)[name = tensor("op_1206")]; - tensor var_1208 = sigmoid(x = var_1206_1)[name = tensor("op_1208")]; - tensor inputs_61 = mul(x = var_1206_0, y = var_1208)[name = tensor("inputs_61")]; - tensor outputs_aug_17_pad_type_0 = const()[name = tensor("outputs_aug_17_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_17_pad_0 = const()[name = tensor("outputs_aug_17_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_17_groups_0 = const()[name = tensor("outputs_aug_17_groups_0"), val = tensor(256)]; - tensor outputs_aug_17_strides_0 = const()[name = tensor("outputs_aug_17_strides_0"), val = tensor([1])]; - tensor outputs_aug_17_dilations_0 = const()[name = tensor("outputs_aug_17_dilations_0"), val = tensor([1])]; - tensor outputs_aug_17 = conv(dilations = outputs_aug_17_dilations_0, groups = outputs_aug_17_groups_0, pad = outputs_aug_17_pad_0, pad_type = outputs_aug_17_pad_type_0, strides = outputs_aug_17_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_61)[name = tensor("outputs_aug_17")]; - tensor input_165_begin_0 = const()[name = tensor("input_165_begin_0"), val = tensor([0, 0, 0])]; - tensor input_165_end_0 = const()[name = tensor("input_165_end_0"), val = tensor([1, 256, 16])]; - tensor input_165_end_mask_0 = const()[name = tensor("input_165_end_mask_0"), val = tensor([true, true, false])]; - tensor input_165 = slice_by_index(begin = input_165_begin_0, end = input_165_end_0, end_mask = input_165_end_mask_0, x = outputs_aug_17)[name = tensor("input_165")]; - tensor inputs_63 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_165)[name = tensor("inputs_63")]; - tensor input_167 = silu(x = inputs_63)[name = tensor("input_167")]; - tensor input_169_pad_type_0 = const()[name = tensor("input_169_pad_type_0"), val = tensor("valid")]; - tensor input_169_strides_0 = const()[name = tensor("input_169_strides_0"), val = tensor([1])]; - tensor input_169_pad_0 = const()[name = tensor("input_169_pad_0"), val = tensor([0, 0])]; - tensor input_169_dilations_0 = const()[name = tensor("input_169_dilations_0"), val = tensor([1])]; - tensor input_169_groups_0 = const()[name = tensor("input_169_groups_0"), val = tensor(1)]; - tensor input_169 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_169_dilations_0, groups = input_169_groups_0, pad = input_169_pad_0, pad_type = input_169_pad_type_0, strides = input_169_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_167)[name = tensor("input_169")]; - tensor conv_out_17_perm_0 = const()[name = tensor("conv_out_17_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1239_begin_0 = const()[name = tensor("op_1239_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1239_end_0 = const()[name = tensor("op_1239_end_0"), val = tensor([1, 16, 256])]; - tensor var_1239_end_mask_0 = const()[name = tensor("op_1239_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_17 = transpose(perm = conv_out_17_perm_0, x = input_169)[name = tensor("transpose_41")]; - tensor var_1239 = slice_by_index(begin = var_1239_begin_0, end = var_1239_end_0, end_mask = var_1239_end_mask_0, x = conv_out_17)[name = tensor("op_1239")]; - tensor var_1242_interleave_0 = const()[name = tensor("op_1242_interleave_0"), val = tensor(false)]; - tensor var_1242 = concat(axis = var_33, interleave = var_1242_interleave_0, values = (var_1071, var_1155, var_1239))[name = tensor("op_1242")]; - tensor input_171 = add(x = x_23, y = var_1242)[name = tensor("input_171")]; - tensor input_173_axes_0 = const()[name = tensor("input_173_axes_0"), val = tensor([-1])]; - tensor input_173 = layer_norm(axes = input_173_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_171)[name = tensor("input_173")]; - tensor inputs_65 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_173)[name = tensor("linear_26")]; - tensor input_175 = silu(x = inputs_65)[name = tensor("input_175")]; - tensor input_179 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_175)[name = tensor("linear_27")]; - tensor var_1265 = const()[name = tensor("op_1265"), val = tensor(0x1p-1)]; - tensor var_1266 = mul(x = input_179, y = var_1265)[name = tensor("op_1266")]; - tensor input_181 = add(x = var_1266, y = input_171)[name = tensor("input_181")]; - tensor input_183_axes_0 = const()[name = tensor("input_183_axes_0"), val = tensor([-1])]; - tensor input_183 = layer_norm(axes = input_183_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_181)[name = tensor("input_183")]; - tensor input_185_axes_0 = const()[name = tensor("input_185_axes_0"), val = tensor([-1])]; - tensor input_185 = layer_norm(axes = input_185_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_183)[name = tensor("input_185")]; - tensor inputs_67 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_185)[name = tensor("linear_28")]; - tensor input_187 = silu(x = inputs_67)[name = tensor("input_187")]; - tensor input_191 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_187)[name = tensor("linear_29")]; - tensor var_1295 = const()[name = tensor("op_1295"), val = tensor(0x1p-1)]; - tensor var_1296 = mul(x = input_191, y = var_1295)[name = tensor("op_1296")]; - tensor input_193 = add(x = var_1296, y = input_183)[name = tensor("input_193")]; - tensor x_31_axes_0 = const()[name = tensor("x_31_axes_0"), val = tensor([-1])]; - tensor x_31 = layer_norm(axes = x_31_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_193)[name = tensor("x_31")]; + tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_836)[name = tensor("x_expanded_17")]; + tensor var_850 = sub(x = var_17, y = fill_mask_17)[name = tensor("op_850")]; + tensor var_851 = mul(x = window_21, y = var_850)[name = tensor("op_851")]; + tensor var_852 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_852")]; + tensor filled_window_17 = add(x = var_851, y = var_852)[name = tensor("filled_window_17")]; + tensor var_854 = greater_equal(x = clip_15, y = var_16)[name = tensor("op_854")]; + tensor var_855_dtype_0 = const()[name = tensor("op_855_dtype_0"), val = tensor("fp32")]; + tensor var_856 = const()[name = tensor("op_856"), val = tensor([1, 1, 1])]; + tensor var_855 = cast(dtype = var_855_dtype_0, x = var_854)[name = tensor("cast_142")]; + tensor use_roll_17 = reshape(shape = var_856, x = var_855)[name = tensor("use_roll_17")]; + tensor var_858 = sub(x = var_17, y = use_roll_17)[name = tensor("op_858")]; + tensor var_859 = mul(x = filled_window_17, y = var_858)[name = tensor("op_859")]; + tensor var_860 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_860")]; + tensor window_23 = add(x = var_859, y = var_860)[name = tensor("window_23")]; + tensor var_862 = const()[name = tensor("op_862"), val = tensor(0x1p+0)]; + tensor var_863 = add(x = clip_15, y = var_862)[name = tensor("op_863")]; + tensor clip_17 = clip(alpha = var_21, beta = var_16, x = var_863)[name = tensor("clip_17")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_30, interleave = input_101_interleave_0, values = (window_19, window_21, window_23))[name = tensor("input_101")]; + tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; + tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; + tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_23_pad_type_0 = const()[name = tensor("inputs_23_pad_type_0"), val = tensor("valid")]; + tensor inputs_23_strides_0 = const()[name = tensor("inputs_23_strides_0"), val = tensor([1])]; + tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; + tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; + tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; + tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; + tensor var_886_split_sizes_0 = const()[name = tensor("op_886_split_sizes_0"), val = tensor([256, 256])]; + tensor var_886_axis_0 = const()[name = tensor("op_886_axis_0"), val = tensor(1)]; + tensor var_886_0, tensor var_886_1 = split(axis = var_886_axis_0, split_sizes = var_886_split_sizes_0, x = inputs_23)[name = tensor("op_886")]; + tensor var_888 = sigmoid(x = var_886_1)[name = tensor("op_888")]; + tensor inputs_25 = mul(x = var_886_0, y = var_888)[name = tensor("inputs_25")]; + tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; + tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; + tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; + tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_25)[name = tensor("outputs_aug_5")]; + tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; + tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([3, 256, 16])]; + tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; + tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_5)[name = tensor("input_105")]; + tensor inputs_27 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_105)[name = tensor("inputs_27")]; + tensor input_107 = silu(x = inputs_27)[name = tensor("input_107")]; + tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; + tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; + tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; + tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; + tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; + tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; + tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; + tensor var_919_begin_0 = const()[name = tensor("op_919_begin_0"), val = tensor([0, -1, 0])]; + tensor var_919_end_0 = const()[name = tensor("op_919_end_0"), val = tensor([3, 16, 256])]; + tensor var_919_end_mask_0 = const()[name = tensor("op_919_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_919 = slice_by_index(begin = var_919_begin_0, end = var_919_end_0, end_mask = var_919_end_mask_0, x = conv_out_5)[name = tensor("op_919")]; + tensor var_921_perm_0 = const()[name = tensor("op_921_perm_0"), val = tensor([1, 0, 2])]; + tensor var_921 = transpose(perm = var_921_perm_0, x = var_919)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_921)[name = tensor("input_111")]; + tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; + tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; + tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; + tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; + tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; + tensor var_944 = const()[name = tensor("op_944"), val = tensor(0x1p-1)]; + tensor var_945 = mul(x = input_119, y = var_944)[name = tensor("op_945")]; + tensor input_121 = add(x = var_945, y = input_111)[name = tensor("input_121")]; + tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; + tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; + tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; + tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; + tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; + tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; + tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; + tensor var_974 = const()[name = tensor("op_974"), val = tensor(0x1p-1)]; + tensor var_975 = mul(x = input_131, y = var_974)[name = tensor("op_975")]; + tensor input_133 = add(x = var_975, y = input_123)[name = tensor("input_133")]; + tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; + tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; tensor prev_kv_7_end_0 = const()[name = tensor("prev_kv_7_end_0"), val = tensor([4, 1, 4, 64, 64])]; tensor prev_kv_7_end_mask_0 = const()[name = tensor("prev_kv_7_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1203,309 +976,233 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_1310 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_31)[name = tensor("linear_30")]; - tensor var_1311 = const()[name = tensor("op_1311"), val = tensor([1, 3, 4, 64])]; - tensor var_1312 = reshape(shape = var_1311, x = var_1310)[name = tensor("op_1312")]; + tensor var_989 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_990 = const()[name = tensor("op_990"), val = tensor([1, 3, 4, 64])]; + tensor var_991 = reshape(shape = var_990, x = var_989)[name = tensor("op_991")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1316 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_31)[name = tensor("linear_31")]; - tensor var_1317 = const()[name = tensor("op_1317"), val = tensor(0x1p-3)]; - tensor var_1318 = mul(x = var_1316, y = var_1317)[name = tensor("op_1318")]; - tensor var_1319 = const()[name = tensor("op_1319"), val = tensor([1, 3, 4, 64])]; - tensor var_1320 = reshape(shape = var_1319, x = var_1318)[name = tensor("op_1320")]; + tensor var_995 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_996 = const()[name = tensor("op_996"), val = tensor(0x1p-3)]; + tensor var_997 = mul(x = var_995, y = var_996)[name = tensor("op_997")]; + tensor var_998 = const()[name = tensor("op_998"), val = tensor([1, 3, 4, 64])]; + tensor var_999 = reshape(shape = var_998, x = var_997)[name = tensor("op_999")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1324 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_31)[name = tensor("linear_32")]; - tensor var_1325 = const()[name = tensor("op_1325"), val = tensor([1, 3, 4, 64])]; - tensor var_1326 = reshape(shape = var_1325, x = var_1324)[name = tensor("op_1326")]; + tensor var_1003 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_1004 = const()[name = tensor("op_1004"), val = tensor([1, 3, 4, 64])]; + tensor var_1005 = reshape(shape = var_1004, x = var_1003)[name = tensor("op_1005")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_197 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_31)[name = tensor("linear_33")]; + tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; tensor s_t_7 = add(x = prev_scale_7, y = encoder__t_index)[name = tensor("s_t_7")]; tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_1320)[name = tensor("transpose_39")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_1312)[name = tensor("transpose_40")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_999)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_991)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_1336 = const()[name = tensor("op_1336"), val = tensor([3, 1])]; - tensor var_1337 = reshape(shape = var_1336, x = sqrt_s_t_7)[name = tensor("op_1337")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_1337)[name = tensor("M_7")]; - tensor var_1339 = mul(x = qk_7, y = M_7)[name = tensor("op_1339")]; + tensor var_1015 = const()[name = tensor("op_1015"), val = tensor([3, 1])]; + tensor var_1016 = reshape(shape = var_1015, x = sqrt_s_t_7)[name = tensor("op_1016")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_1016)[name = tensor("M_7")]; + tensor var_1018 = mul(x = qk_7, y = M_7)[name = tensor("op_1018")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_1326)[name = tensor("transpose_38")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1339, y = v_7)[name = tensor("inner_7")]; - tensor var_1341_transpose_x_0 = const()[name = tensor("op_1341_transpose_x_0"), val = tensor(false)]; - tensor var_1341_transpose_y_0 = const()[name = tensor("op_1341_transpose_y_0"), val = tensor(false)]; - tensor var_1341 = matmul(transpose_x = var_1341_transpose_x_0, transpose_y = var_1341_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1341")]; - tensor var_1342 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1342")]; - tensor var_1343 = const()[name = tensor("op_1343"), val = tensor([1, 1, 3, 1])]; - tensor var_1344 = reshape(shape = var_1343, x = var_1342)[name = tensor("op_1344")]; - tensor cross_7 = mul(x = var_1341, y = var_1344)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_1005)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1018, y = v_7)[name = tensor("inner_7")]; + tensor var_1020_transpose_x_0 = const()[name = tensor("op_1020_transpose_x_0"), val = tensor(false)]; + tensor var_1020_transpose_y_0 = const()[name = tensor("op_1020_transpose_y_0"), val = tensor(false)]; + tensor var_1020 = matmul(transpose_x = var_1020_transpose_x_0, transpose_y = var_1020_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1020")]; + tensor var_1021 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1021")]; + tensor var_1022 = const()[name = tensor("op_1022"), val = tensor([1, 1, 3, 1])]; + tensor var_1023 = reshape(shape = var_1022, x = var_1021)[name = tensor("op_1023")]; + tensor cross_7 = mul(x = var_1020, y = var_1023)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_1347 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1347")]; - tensor var_1349_transpose_x_1 = const()[name = tensor("op_1349_transpose_x_1"), val = tensor(true)]; - tensor var_1349_transpose_y_1 = const()[name = tensor("op_1349_transpose_y_1"), val = tensor(false)]; - tensor var_1349 = matmul(transpose_x = var_1349_transpose_x_1, transpose_y = var_1349_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1349")]; - tensor new_kv_unnorm_7 = add(x = var_1347, y = var_1349)[name = tensor("new_kv_unnorm_7")]; - tensor var_1351 = const()[name = tensor("op_1351"), val = tensor(0x1.8p+1)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_1351)[name = tensor("new_scale_7")]; - tensor var_1353 = sqrt(x = new_scale_7)[name = tensor("op_1353")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1353)[name = tensor("nkv_1")]; - tensor var_1355_perm_0 = const()[name = tensor("op_1355_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1026 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1026")]; + tensor var_1028_transpose_x_1 = const()[name = tensor("op_1028_transpose_x_1"), val = tensor(true)]; + tensor var_1028_transpose_y_1 = const()[name = tensor("op_1028_transpose_y_1"), val = tensor(false)]; + tensor var_1028 = matmul(transpose_x = var_1028_transpose_x_1, transpose_y = var_1028_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1028")]; + tensor new_kv_unnorm_7 = add(x = var_1026, y = var_1028)[name = tensor("new_kv_unnorm_7")]; + tensor var_1030 = const()[name = tensor("op_1030"), val = tensor(0x1.8p+1)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_1030)[name = tensor("new_scale_7")]; + tensor var_1032 = sqrt(x = new_scale_7)[name = tensor("op_1032")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1032)[name = tensor("nkv_1")]; + tensor var_1034_perm_0 = const()[name = tensor("op_1034_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_1355 = transpose(perm = var_1355_perm_0, x = out_19)[name = tensor("transpose_37")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1355)[name = tensor("out_21")]; - tensor var_1359 = const()[name = tensor("op_1359"), val = tensor([1, 3, 256])]; - tensor out_23 = reshape(shape = var_1359, x = out_21)[name = tensor("out_23")]; - tensor var_1361 = silu(x = input_197)[name = tensor("op_1361")]; - tensor input_199 = mul(x = var_1361, y = out_23)[name = tensor("input_199")]; - tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_199)[name = tensor("linear_34")]; - tensor x_33 = add(x = input_193, y = ret_out_7)[name = tensor("x_33")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor var_1034 = transpose(perm = var_1034_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1034)[name = tensor("out_21")]; + tensor var_1038 = const()[name = tensor("op_1038"), val = tensor([1, 3, 256])]; + tensor out_23 = reshape(shape = var_1038, x = out_21)[name = tensor("out_23")]; + tensor var_1040 = silu(x = input_137)[name = tensor("op_1040")]; + tensor input_139 = mul(x = var_1040, y = out_23)[name = tensor("input_139")]; + tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; + tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; + tensor window_25_begin_0 = const()[name = tensor("window_25_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_25_end_0 = const()[name = tensor("window_25_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_25_end_mask_0 = const()[name = tensor("window_25_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_25_squeeze_mask_0 = const()[name = tensor("window_25_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_25 = slice_by_index(begin = window_25_begin_0, end = window_25_end_0, end_mask = window_25_end_mask_0, squeeze_mask = window_25_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_25")]; tensor filled_19_begin_0 = const()[name = tensor("filled_19_begin_0"), val = tensor([3, 0])]; tensor filled_19_end_0 = const()[name = tensor("filled_19_end_0"), val = tensor([4, 1])]; tensor filled_19_end_mask_0 = const()[name = tensor("filled_19_end_mask_0"), val = tensor([false, true])]; tensor filled_19_squeeze_mask_0 = const()[name = tensor("filled_19_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_19 = slice_by_index(begin = filled_19_begin_0, end = filled_19_end_0, end_mask = filled_19_end_mask_0, squeeze_mask = filled_19_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_19")]; - tensor var_1370_begin_0 = const()[name = tensor("op_1370_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1370_end_0 = const()[name = tensor("op_1370_end_0"), val = tensor([1, 1, 256])]; - tensor var_1370_end_mask_0 = const()[name = tensor("op_1370_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1370 = slice_by_index(begin = var_1370_begin_0, end = var_1370_end_0, end_mask = var_1370_end_mask_0, x = x_33)[name = tensor("op_1370")]; - tensor var_1373_begin_0 = const()[name = tensor("op_1373_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1373_end_0 = const()[name = tensor("op_1373_end_0"), val = tensor([1, 16, 256])]; - tensor var_1373_end_mask_0 = const()[name = tensor("op_1373_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1373 = slice_by_index(begin = var_1373_begin_0, end = var_1373_end_0, end_mask = var_1373_end_mask_0, x = window)[name = tensor("op_1373")]; + tensor var_1049_begin_0 = const()[name = tensor("op_1049_begin_0"), val = tensor([0, 0, 0])]; + tensor var_1049_end_0 = const()[name = tensor("op_1049_end_0"), val = tensor([1, 1, 256])]; + tensor var_1049_end_mask_0 = const()[name = tensor("op_1049_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1049 = slice_by_index(begin = var_1049_begin_0, end = var_1049_end_0, end_mask = var_1049_end_mask_0, x = x_21)[name = tensor("op_1049")]; + tensor var_1052_begin_0 = const()[name = tensor("op_1052_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1052_end_0 = const()[name = tensor("op_1052_end_0"), val = tensor([1, 16, 256])]; + tensor var_1052_end_mask_0 = const()[name = tensor("op_1052_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1052 = slice_by_index(begin = var_1052_begin_0, end = var_1052_end_0, end_mask = var_1052_end_mask_0, x = window_25)[name = tensor("op_1052")]; tensor rolled_19_interleave_0 = const()[name = tensor("rolled_19_interleave_0"), val = tensor(false)]; - tensor rolled_19 = concat(axis = var_33, interleave = rolled_19_interleave_0, values = (var_1373, var_1370))[name = tensor("rolled_19")]; + tensor rolled_19 = concat(axis = var_33, interleave = rolled_19_interleave_0, values = (var_1052, var_1049))[name = tensor("rolled_19")]; tensor clip_18 = clip(alpha = var_21, beta = var_20, x = filled_19)[name = tensor("clip_18")]; - tensor var_1378 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_1378")]; - tensor var_1379_dtype_0 = const()[name = tensor("op_1379_dtype_0"), val = tensor("fp32")]; - tensor var_1380 = const()[name = tensor("op_1380"), val = tensor([1, 16, 1])]; - tensor var_1379 = cast(dtype = var_1379_dtype_0, x = var_1378)[name = tensor("cast_141")]; - tensor fill_mask_19 = reshape(shape = var_1380, x = var_1379)[name = tensor("fill_mask_19")]; + tensor var_1057 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_1057")]; + tensor var_1058_dtype_0 = const()[name = tensor("op_1058_dtype_0"), val = tensor("fp32")]; + tensor var_1059 = const()[name = tensor("op_1059"), val = tensor([1, 16, 1])]; + tensor var_1058 = cast(dtype = var_1058_dtype_0, x = var_1057)[name = tensor("cast_141")]; + tensor fill_mask_19 = reshape(shape = var_1059, x = var_1058)[name = tensor("fill_mask_19")]; tensor x_expanded_19_reps_0 = const()[name = tensor("x_expanded_19_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_1370)[name = tensor("x_expanded_19")]; - tensor var_1384 = sub(x = var_17, y = fill_mask_19)[name = tensor("op_1384")]; - tensor var_1385 = mul(x = window, y = var_1384)[name = tensor("op_1385")]; - tensor var_1386 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_1386")]; - tensor filled_window_19 = add(x = var_1385, y = var_1386)[name = tensor("filled_window_19")]; - tensor var_1388 = greater_equal(x = filled_19, y = var_16)[name = tensor("op_1388")]; - tensor var_1389_dtype_0 = const()[name = tensor("op_1389_dtype_0"), val = tensor("fp32")]; - tensor var_1390 = const()[name = tensor("op_1390"), val = tensor([1, 1, 1])]; - tensor var_1389 = cast(dtype = var_1389_dtype_0, x = var_1388)[name = tensor("cast_140")]; - tensor use_roll_19 = reshape(shape = var_1390, x = var_1389)[name = tensor("use_roll_19")]; - tensor var_1392 = sub(x = var_17, y = use_roll_19)[name = tensor("op_1392")]; - tensor var_1393 = mul(x = filled_window_19, y = var_1392)[name = tensor("op_1393")]; - tensor var_1394 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_1394")]; - tensor input_201 = add(x = var_1393, y = var_1394)[name = tensor("input_201")]; - tensor var_1396 = const()[name = tensor("op_1396"), val = tensor(0x1p+0)]; - tensor var_1397 = add(x = filled_19, y = var_1396)[name = tensor("op_1397")]; - tensor clip_19 = clip(alpha = var_21, beta = var_16, x = var_1397)[name = tensor("clip_19")]; - tensor x_35_axes_0 = const()[name = tensor("x_35_axes_0"), val = tensor([-1])]; - tensor x_35 = layer_norm(axes = x_35_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_201)[name = tensor("x_35")]; - tensor input_203_perm_0 = const()[name = tensor("input_203_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_69_pad_type_0 = const()[name = tensor("inputs_69_pad_type_0"), val = tensor("valid")]; - tensor inputs_69_strides_0 = const()[name = tensor("inputs_69_strides_0"), val = tensor([1])]; - tensor inputs_69_pad_0 = const()[name = tensor("inputs_69_pad_0"), val = tensor([0, 0])]; - tensor inputs_69_dilations_0 = const()[name = tensor("inputs_69_dilations_0"), val = tensor([1])]; - tensor inputs_69_groups_0 = const()[name = tensor("inputs_69_groups_0"), val = tensor(1)]; - tensor input_203 = transpose(perm = input_203_perm_0, x = x_35)[name = tensor("transpose_36")]; - tensor inputs_69 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_69_dilations_0, groups = inputs_69_groups_0, pad = inputs_69_pad_0, pad_type = inputs_69_pad_type_0, strides = inputs_69_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_203)[name = tensor("inputs_69")]; - tensor var_1418_split_sizes_0 = const()[name = tensor("op_1418_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1418_axis_0 = const()[name = tensor("op_1418_axis_0"), val = tensor(1)]; - tensor var_1418_0, tensor var_1418_1 = split(axis = var_1418_axis_0, split_sizes = var_1418_split_sizes_0, x = inputs_69)[name = tensor("op_1418")]; - tensor var_1420 = sigmoid(x = var_1418_1)[name = tensor("op_1420")]; - tensor inputs_71 = mul(x = var_1418_0, y = var_1420)[name = tensor("inputs_71")]; - tensor outputs_aug_19_pad_type_0 = const()[name = tensor("outputs_aug_19_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_19_pad_0 = const()[name = tensor("outputs_aug_19_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_19_groups_0 = const()[name = tensor("outputs_aug_19_groups_0"), val = tensor(256)]; - tensor outputs_aug_19_strides_0 = const()[name = tensor("outputs_aug_19_strides_0"), val = tensor([1])]; - tensor outputs_aug_19_dilations_0 = const()[name = tensor("outputs_aug_19_dilations_0"), val = tensor([1])]; - tensor outputs_aug_19 = conv(dilations = outputs_aug_19_dilations_0, groups = outputs_aug_19_groups_0, pad = outputs_aug_19_pad_0, pad_type = outputs_aug_19_pad_type_0, strides = outputs_aug_19_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_71)[name = tensor("outputs_aug_19")]; - tensor input_205_begin_0 = const()[name = tensor("input_205_begin_0"), val = tensor([0, 0, 0])]; - tensor input_205_end_0 = const()[name = tensor("input_205_end_0"), val = tensor([1, 256, 16])]; - tensor input_205_end_mask_0 = const()[name = tensor("input_205_end_mask_0"), val = tensor([true, true, false])]; - tensor input_205 = slice_by_index(begin = input_205_begin_0, end = input_205_end_0, end_mask = input_205_end_mask_0, x = outputs_aug_19)[name = tensor("input_205")]; - tensor inputs_73 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_205)[name = tensor("inputs_73")]; - tensor input_207 = silu(x = inputs_73)[name = tensor("input_207")]; - tensor input_209_pad_type_0 = const()[name = tensor("input_209_pad_type_0"), val = tensor("valid")]; - tensor input_209_strides_0 = const()[name = tensor("input_209_strides_0"), val = tensor([1])]; - tensor input_209_pad_0 = const()[name = tensor("input_209_pad_0"), val = tensor([0, 0])]; - tensor input_209_dilations_0 = const()[name = tensor("input_209_dilations_0"), val = tensor([1])]; - tensor input_209_groups_0 = const()[name = tensor("input_209_groups_0"), val = tensor(1)]; - tensor input_209 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_209_dilations_0, groups = input_209_groups_0, pad = input_209_pad_0, pad_type = input_209_pad_type_0, strides = input_209_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_207)[name = tensor("input_209")]; - tensor conv_out_19_perm_0 = const()[name = tensor("conv_out_19_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1451_begin_0 = const()[name = tensor("op_1451_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1451_end_0 = const()[name = tensor("op_1451_end_0"), val = tensor([1, 16, 256])]; - tensor var_1451_end_mask_0 = const()[name = tensor("op_1451_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_19 = transpose(perm = conv_out_19_perm_0, x = input_209)[name = tensor("transpose_35")]; - tensor var_1451 = slice_by_index(begin = var_1451_begin_0, end = var_1451_end_0, end_mask = var_1451_end_mask_0, x = conv_out_19)[name = tensor("op_1451")]; - tensor var_1454_begin_0 = const()[name = tensor("op_1454_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1454_end_0 = const()[name = tensor("op_1454_end_0"), val = tensor([1, 2, 256])]; - tensor var_1454_end_mask_0 = const()[name = tensor("op_1454_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1454 = slice_by_index(begin = var_1454_begin_0, end = var_1454_end_0, end_mask = var_1454_end_mask_0, x = x_33)[name = tensor("op_1454")]; - tensor var_1457_begin_0 = const()[name = tensor("op_1457_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1457_end_0 = const()[name = tensor("op_1457_end_0"), val = tensor([1, 16, 256])]; - tensor var_1457_end_mask_0 = const()[name = tensor("op_1457_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1457 = slice_by_index(begin = var_1457_begin_0, end = var_1457_end_0, end_mask = var_1457_end_mask_0, x = input_201)[name = tensor("op_1457")]; + tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_1049)[name = tensor("x_expanded_19")]; + tensor var_1063 = sub(x = var_17, y = fill_mask_19)[name = tensor("op_1063")]; + tensor var_1064 = mul(x = window_25, y = var_1063)[name = tensor("op_1064")]; + tensor var_1065 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_1065")]; + tensor filled_window_19 = add(x = var_1064, y = var_1065)[name = tensor("filled_window_19")]; + tensor var_1067 = greater_equal(x = filled_19, y = var_16)[name = tensor("op_1067")]; + tensor var_1068_dtype_0 = const()[name = tensor("op_1068_dtype_0"), val = tensor("fp32")]; + tensor var_1069 = const()[name = tensor("op_1069"), val = tensor([1, 1, 1])]; + tensor var_1068 = cast(dtype = var_1068_dtype_0, x = var_1067)[name = tensor("cast_140")]; + tensor use_roll_19 = reshape(shape = var_1069, x = var_1068)[name = tensor("use_roll_19")]; + tensor var_1071 = sub(x = var_17, y = use_roll_19)[name = tensor("op_1071")]; + tensor var_1072 = mul(x = filled_window_19, y = var_1071)[name = tensor("op_1072")]; + tensor var_1073 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_1073")]; + tensor window_27 = add(x = var_1072, y = var_1073)[name = tensor("window_27")]; + tensor var_1075 = const()[name = tensor("op_1075"), val = tensor(0x1p+0)]; + tensor var_1076 = add(x = filled_19, y = var_1075)[name = tensor("op_1076")]; + tensor clip_19 = clip(alpha = var_21, beta = var_16, x = var_1076)[name = tensor("clip_19")]; + tensor var_1079_begin_0 = const()[name = tensor("op_1079_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1079_end_0 = const()[name = tensor("op_1079_end_0"), val = tensor([1, 2, 256])]; + tensor var_1079_end_mask_0 = const()[name = tensor("op_1079_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1079 = slice_by_index(begin = var_1079_begin_0, end = var_1079_end_0, end_mask = var_1079_end_mask_0, x = x_21)[name = tensor("op_1079")]; + tensor var_1082_begin_0 = const()[name = tensor("op_1082_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1082_end_0 = const()[name = tensor("op_1082_end_0"), val = tensor([1, 16, 256])]; + tensor var_1082_end_mask_0 = const()[name = tensor("op_1082_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1082 = slice_by_index(begin = var_1082_begin_0, end = var_1082_end_0, end_mask = var_1082_end_mask_0, x = window_27)[name = tensor("op_1082")]; tensor rolled_21_interleave_0 = const()[name = tensor("rolled_21_interleave_0"), val = tensor(false)]; - tensor rolled_21 = concat(axis = var_33, interleave = rolled_21_interleave_0, values = (var_1457, var_1454))[name = tensor("rolled_21")]; + tensor rolled_21 = concat(axis = var_33, interleave = rolled_21_interleave_0, values = (var_1082, var_1079))[name = tensor("rolled_21")]; tensor clip_20 = clip(alpha = var_21, beta = var_20, x = clip_19)[name = tensor("clip_20")]; - tensor var_1462 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_1462")]; - tensor var_1463_dtype_0 = const()[name = tensor("op_1463_dtype_0"), val = tensor("fp32")]; - tensor var_1464 = const()[name = tensor("op_1464"), val = tensor([1, 16, 1])]; - tensor var_1463 = cast(dtype = var_1463_dtype_0, x = var_1462)[name = tensor("cast_139")]; - tensor fill_mask_21 = reshape(shape = var_1464, x = var_1463)[name = tensor("fill_mask_21")]; + tensor var_1087 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_1087")]; + tensor var_1088_dtype_0 = const()[name = tensor("op_1088_dtype_0"), val = tensor("fp32")]; + tensor var_1089 = const()[name = tensor("op_1089"), val = tensor([1, 16, 1])]; + tensor var_1088 = cast(dtype = var_1088_dtype_0, x = var_1087)[name = tensor("cast_139")]; + tensor fill_mask_21 = reshape(shape = var_1089, x = var_1088)[name = tensor("fill_mask_21")]; tensor x_expanded_21_reps_0 = const()[name = tensor("x_expanded_21_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_1454)[name = tensor("x_expanded_21")]; - tensor var_1468 = sub(x = var_17, y = fill_mask_21)[name = tensor("op_1468")]; - tensor var_1469 = mul(x = input_201, y = var_1468)[name = tensor("op_1469")]; - tensor var_1470 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_1470")]; - tensor filled_window_21 = add(x = var_1469, y = var_1470)[name = tensor("filled_window_21")]; - tensor var_1472 = greater_equal(x = clip_19, y = var_16)[name = tensor("op_1472")]; - tensor var_1473_dtype_0 = const()[name = tensor("op_1473_dtype_0"), val = tensor("fp32")]; - tensor var_1474 = const()[name = tensor("op_1474"), val = tensor([1, 1, 1])]; - tensor var_1473 = cast(dtype = var_1473_dtype_0, x = var_1472)[name = tensor("cast_138")]; - tensor use_roll_21 = reshape(shape = var_1474, x = var_1473)[name = tensor("use_roll_21")]; - tensor var_1476 = sub(x = var_17, y = use_roll_21)[name = tensor("op_1476")]; - tensor var_1477 = mul(x = filled_window_21, y = var_1476)[name = tensor("op_1477")]; - tensor var_1478 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_1478")]; - tensor input_211 = add(x = var_1477, y = var_1478)[name = tensor("input_211")]; - tensor var_1480 = const()[name = tensor("op_1480"), val = tensor(0x1p+0)]; - tensor var_1481 = add(x = clip_19, y = var_1480)[name = tensor("op_1481")]; - tensor clip_21 = clip(alpha = var_21, beta = var_16, x = var_1481)[name = tensor("clip_21")]; - tensor x_37_axes_0 = const()[name = tensor("x_37_axes_0"), val = tensor([-1])]; - tensor x_37 = layer_norm(axes = x_37_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_211)[name = tensor("x_37")]; - tensor input_213_perm_0 = const()[name = tensor("input_213_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_75_pad_type_0 = const()[name = tensor("inputs_75_pad_type_0"), val = tensor("valid")]; - tensor inputs_75_strides_0 = const()[name = tensor("inputs_75_strides_0"), val = tensor([1])]; - tensor inputs_75_pad_0 = const()[name = tensor("inputs_75_pad_0"), val = tensor([0, 0])]; - tensor inputs_75_dilations_0 = const()[name = tensor("inputs_75_dilations_0"), val = tensor([1])]; - tensor inputs_75_groups_0 = const()[name = tensor("inputs_75_groups_0"), val = tensor(1)]; - tensor input_213 = transpose(perm = input_213_perm_0, x = x_37)[name = tensor("transpose_34")]; - tensor inputs_75 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_75_dilations_0, groups = inputs_75_groups_0, pad = inputs_75_pad_0, pad_type = inputs_75_pad_type_0, strides = inputs_75_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_213)[name = tensor("inputs_75")]; - tensor var_1502_split_sizes_0 = const()[name = tensor("op_1502_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1502_axis_0 = const()[name = tensor("op_1502_axis_0"), val = tensor(1)]; - tensor var_1502_0, tensor var_1502_1 = split(axis = var_1502_axis_0, split_sizes = var_1502_split_sizes_0, x = inputs_75)[name = tensor("op_1502")]; - tensor var_1504 = sigmoid(x = var_1502_1)[name = tensor("op_1504")]; - tensor inputs_77 = mul(x = var_1502_0, y = var_1504)[name = tensor("inputs_77")]; - tensor outputs_aug_21_pad_type_0 = const()[name = tensor("outputs_aug_21_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_21_pad_0 = const()[name = tensor("outputs_aug_21_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_21_groups_0 = const()[name = tensor("outputs_aug_21_groups_0"), val = tensor(256)]; - tensor outputs_aug_21_strides_0 = const()[name = tensor("outputs_aug_21_strides_0"), val = tensor([1])]; - tensor outputs_aug_21_dilations_0 = const()[name = tensor("outputs_aug_21_dilations_0"), val = tensor([1])]; - tensor outputs_aug_21 = conv(dilations = outputs_aug_21_dilations_0, groups = outputs_aug_21_groups_0, pad = outputs_aug_21_pad_0, pad_type = outputs_aug_21_pad_type_0, strides = outputs_aug_21_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_77)[name = tensor("outputs_aug_21")]; - tensor input_215_begin_0 = const()[name = tensor("input_215_begin_0"), val = tensor([0, 0, 0])]; - tensor input_215_end_0 = const()[name = tensor("input_215_end_0"), val = tensor([1, 256, 16])]; - tensor input_215_end_mask_0 = const()[name = tensor("input_215_end_mask_0"), val = tensor([true, true, false])]; - tensor input_215 = slice_by_index(begin = input_215_begin_0, end = input_215_end_0, end_mask = input_215_end_mask_0, x = outputs_aug_21)[name = tensor("input_215")]; - tensor inputs_79 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_215)[name = tensor("inputs_79")]; - tensor input_217 = silu(x = inputs_79)[name = tensor("input_217")]; - tensor input_219_pad_type_0 = const()[name = tensor("input_219_pad_type_0"), val = tensor("valid")]; - tensor input_219_strides_0 = const()[name = tensor("input_219_strides_0"), val = tensor([1])]; - tensor input_219_pad_0 = const()[name = tensor("input_219_pad_0"), val = tensor([0, 0])]; - tensor input_219_dilations_0 = const()[name = tensor("input_219_dilations_0"), val = tensor([1])]; - tensor input_219_groups_0 = const()[name = tensor("input_219_groups_0"), val = tensor(1)]; - tensor input_219 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_219_dilations_0, groups = input_219_groups_0, pad = input_219_pad_0, pad_type = input_219_pad_type_0, strides = input_219_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_217)[name = tensor("input_219")]; - tensor conv_out_21_perm_0 = const()[name = tensor("conv_out_21_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1535_begin_0 = const()[name = tensor("op_1535_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1535_end_0 = const()[name = tensor("op_1535_end_0"), val = tensor([1, 16, 256])]; - tensor var_1535_end_mask_0 = const()[name = tensor("op_1535_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_21 = transpose(perm = conv_out_21_perm_0, x = input_219)[name = tensor("transpose_33")]; - tensor var_1535 = slice_by_index(begin = var_1535_begin_0, end = var_1535_end_0, end_mask = var_1535_end_mask_0, x = conv_out_21)[name = tensor("op_1535")]; - tensor var_1538_begin_0 = const()[name = tensor("op_1538_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1538_end_0 = const()[name = tensor("op_1538_end_0"), val = tensor([1, 1, 256])]; - tensor var_1538_end_mask_0 = const()[name = tensor("op_1538_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1538 = slice_by_index(begin = var_1538_begin_0, end = var_1538_end_0, end_mask = var_1538_end_mask_0, x = x_33)[name = tensor("op_1538")]; - tensor var_1541_begin_0 = const()[name = tensor("op_1541_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1541_end_0 = const()[name = tensor("op_1541_end_0"), val = tensor([1, 16, 256])]; - tensor var_1541_end_mask_0 = const()[name = tensor("op_1541_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1541 = slice_by_index(begin = var_1541_begin_0, end = var_1541_end_0, end_mask = var_1541_end_mask_0, x = input_211)[name = tensor("op_1541")]; + tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_1079)[name = tensor("x_expanded_21")]; + tensor var_1093 = sub(x = var_17, y = fill_mask_21)[name = tensor("op_1093")]; + tensor var_1094 = mul(x = window_27, y = var_1093)[name = tensor("op_1094")]; + tensor var_1095 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_1095")]; + tensor filled_window_21 = add(x = var_1094, y = var_1095)[name = tensor("filled_window_21")]; + tensor var_1097 = greater_equal(x = clip_19, y = var_16)[name = tensor("op_1097")]; + tensor var_1098_dtype_0 = const()[name = tensor("op_1098_dtype_0"), val = tensor("fp32")]; + tensor var_1099 = const()[name = tensor("op_1099"), val = tensor([1, 1, 1])]; + tensor var_1098 = cast(dtype = var_1098_dtype_0, x = var_1097)[name = tensor("cast_138")]; + tensor use_roll_21 = reshape(shape = var_1099, x = var_1098)[name = tensor("use_roll_21")]; + tensor var_1101 = sub(x = var_17, y = use_roll_21)[name = tensor("op_1101")]; + tensor var_1102 = mul(x = filled_window_21, y = var_1101)[name = tensor("op_1102")]; + tensor var_1103 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_1103")]; + tensor window_29 = add(x = var_1102, y = var_1103)[name = tensor("window_29")]; + tensor var_1105 = const()[name = tensor("op_1105"), val = tensor(0x1p+0)]; + tensor var_1106 = add(x = clip_19, y = var_1105)[name = tensor("op_1106")]; + tensor clip_21 = clip(alpha = var_21, beta = var_16, x = var_1106)[name = tensor("clip_21")]; + tensor var_1109_begin_0 = const()[name = tensor("op_1109_begin_0"), val = tensor([0, 2, 0])]; + tensor var_1109_end_0 = const()[name = tensor("op_1109_end_0"), val = tensor([1, 1, 256])]; + tensor var_1109_end_mask_0 = const()[name = tensor("op_1109_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1109 = slice_by_index(begin = var_1109_begin_0, end = var_1109_end_0, end_mask = var_1109_end_mask_0, x = x_21)[name = tensor("op_1109")]; + tensor var_1112_begin_0 = const()[name = tensor("op_1112_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1112_end_0 = const()[name = tensor("op_1112_end_0"), val = tensor([1, 16, 256])]; + tensor var_1112_end_mask_0 = const()[name = tensor("op_1112_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1112 = slice_by_index(begin = var_1112_begin_0, end = var_1112_end_0, end_mask = var_1112_end_mask_0, x = window_29)[name = tensor("op_1112")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1541, var_1538))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1112, var_1109))[name = tensor("rolled")]; tensor clip_22 = clip(alpha = var_21, beta = var_20, x = clip_21)[name = tensor("clip_22")]; - tensor var_1546 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_1546")]; - tensor var_1547_dtype_0 = const()[name = tensor("op_1547_dtype_0"), val = tensor("fp32")]; - tensor var_1548 = const()[name = tensor("op_1548"), val = tensor([1, 16, 1])]; - tensor var_1547 = cast(dtype = var_1547_dtype_0, x = var_1546)[name = tensor("cast_137")]; - tensor fill_mask = reshape(shape = var_1548, x = var_1547)[name = tensor("fill_mask")]; + tensor var_1117 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_1117")]; + tensor var_1118_dtype_0 = const()[name = tensor("op_1118_dtype_0"), val = tensor("fp32")]; + tensor var_1119 = const()[name = tensor("op_1119"), val = tensor([1, 16, 1])]; + tensor var_1118 = cast(dtype = var_1118_dtype_0, x = var_1117)[name = tensor("cast_137")]; + tensor fill_mask = reshape(shape = var_1119, x = var_1118)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1538)[name = tensor("x_expanded")]; - tensor var_1552 = sub(x = var_17, y = fill_mask)[name = tensor("op_1552")]; - tensor var_1553 = mul(x = input_211, y = var_1552)[name = tensor("op_1553")]; - tensor var_1554 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1554")]; - tensor filled_window = add(x = var_1553, y = var_1554)[name = tensor("filled_window")]; - tensor var_1556 = greater_equal(x = clip_21, y = var_16)[name = tensor("op_1556")]; - tensor var_1557_dtype_0 = const()[name = tensor("op_1557_dtype_0"), val = tensor("fp32")]; - tensor var_1558 = const()[name = tensor("op_1558"), val = tensor([1, 1, 1])]; - tensor var_1557 = cast(dtype = var_1557_dtype_0, x = var_1556)[name = tensor("cast_136")]; - tensor use_roll = reshape(shape = var_1558, x = var_1557)[name = tensor("use_roll")]; - tensor var_1560 = sub(x = var_17, y = use_roll)[name = tensor("op_1560")]; - tensor var_1561 = mul(x = filled_window, y = var_1560)[name = tensor("op_1561")]; - tensor var_1562 = mul(x = rolled, y = use_roll)[name = tensor("op_1562")]; - tensor input_221 = add(x = var_1561, y = var_1562)[name = tensor("input_221")]; - tensor var_1564 = const()[name = tensor("op_1564"), val = tensor(0x1p+0)]; - tensor var_1565 = add(x = clip_21, y = var_1564)[name = tensor("op_1565")]; - tensor clip_23 = clip(alpha = var_21, beta = var_16, x = var_1565)[name = tensor("clip_23")]; - tensor x_39_axes_0 = const()[name = tensor("x_39_axes_0"), val = tensor([-1])]; - tensor x_39 = layer_norm(axes = x_39_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_221)[name = tensor("x_39")]; - tensor input_223_perm_0 = const()[name = tensor("input_223_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_81_pad_type_0 = const()[name = tensor("inputs_81_pad_type_0"), val = tensor("valid")]; - tensor inputs_81_strides_0 = const()[name = tensor("inputs_81_strides_0"), val = tensor([1])]; - tensor inputs_81_pad_0 = const()[name = tensor("inputs_81_pad_0"), val = tensor([0, 0])]; - tensor inputs_81_dilations_0 = const()[name = tensor("inputs_81_dilations_0"), val = tensor([1])]; - tensor inputs_81_groups_0 = const()[name = tensor("inputs_81_groups_0"), val = tensor(1)]; - tensor input_223 = transpose(perm = input_223_perm_0, x = x_39)[name = tensor("transpose_32")]; - tensor inputs_81 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_81_dilations_0, groups = inputs_81_groups_0, pad = inputs_81_pad_0, pad_type = inputs_81_pad_type_0, strides = inputs_81_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_223)[name = tensor("inputs_81")]; - tensor var_1586_split_sizes_0 = const()[name = tensor("op_1586_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1586_axis_0 = const()[name = tensor("op_1586_axis_0"), val = tensor(1)]; - tensor var_1586_0, tensor var_1586_1 = split(axis = var_1586_axis_0, split_sizes = var_1586_split_sizes_0, x = inputs_81)[name = tensor("op_1586")]; - tensor var_1588 = sigmoid(x = var_1586_1)[name = tensor("op_1588")]; - tensor inputs_83 = mul(x = var_1586_0, y = var_1588)[name = tensor("inputs_83")]; + tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1109)[name = tensor("x_expanded")]; + tensor var_1123 = sub(x = var_17, y = fill_mask)[name = tensor("op_1123")]; + tensor var_1124 = mul(x = window_29, y = var_1123)[name = tensor("op_1124")]; + tensor var_1125 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1125")]; + tensor filled_window = add(x = var_1124, y = var_1125)[name = tensor("filled_window")]; + tensor var_1127 = greater_equal(x = clip_21, y = var_16)[name = tensor("op_1127")]; + tensor var_1128_dtype_0 = const()[name = tensor("op_1128_dtype_0"), val = tensor("fp32")]; + tensor var_1129 = const()[name = tensor("op_1129"), val = tensor([1, 1, 1])]; + tensor var_1128 = cast(dtype = var_1128_dtype_0, x = var_1127)[name = tensor("cast_136")]; + tensor use_roll = reshape(shape = var_1129, x = var_1128)[name = tensor("use_roll")]; + tensor var_1131 = sub(x = var_17, y = use_roll)[name = tensor("op_1131")]; + tensor var_1132 = mul(x = filled_window, y = var_1131)[name = tensor("op_1132")]; + tensor var_1133 = mul(x = rolled, y = use_roll)[name = tensor("op_1133")]; + tensor window = add(x = var_1132, y = var_1133)[name = tensor("window")]; + tensor var_1135 = const()[name = tensor("op_1135"), val = tensor(0x1p+0)]; + tensor var_1136 = add(x = clip_21, y = var_1135)[name = tensor("op_1136")]; + tensor clip_23 = clip(alpha = var_21, beta = var_16, x = var_1136)[name = tensor("clip_23")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_30, interleave = input_141_interleave_0, values = (window_27, window_29, window))[name = tensor("input_141")]; + tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; + tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; + tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_33_pad_type_0 = const()[name = tensor("inputs_33_pad_type_0"), val = tensor("valid")]; + tensor inputs_33_strides_0 = const()[name = tensor("inputs_33_strides_0"), val = tensor([1])]; + tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; + tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; + tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; + tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; + tensor var_1159_split_sizes_0 = const()[name = tensor("op_1159_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1159_axis_0 = const()[name = tensor("op_1159_axis_0"), val = tensor(1)]; + tensor var_1159_0, tensor var_1159_1 = split(axis = var_1159_axis_0, split_sizes = var_1159_split_sizes_0, x = inputs_33)[name = tensor("op_1159")]; + tensor var_1161 = sigmoid(x = var_1159_1)[name = tensor("op_1161")]; + tensor inputs_35 = mul(x = var_1159_0, y = var_1161)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; tensor outputs_aug_strides_0 = const()[name = tensor("outputs_aug_strides_0"), val = tensor([1])]; tensor outputs_aug_dilations_0 = const()[name = tensor("outputs_aug_dilations_0"), val = tensor([1])]; - tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_83)[name = tensor("outputs_aug")]; - tensor input_225_begin_0 = const()[name = tensor("input_225_begin_0"), val = tensor([0, 0, 0])]; - tensor input_225_end_0 = const()[name = tensor("input_225_end_0"), val = tensor([1, 256, 16])]; - tensor input_225_end_mask_0 = const()[name = tensor("input_225_end_mask_0"), val = tensor([true, true, false])]; - tensor input_225 = slice_by_index(begin = input_225_begin_0, end = input_225_end_0, end_mask = input_225_end_mask_0, x = outputs_aug)[name = tensor("input_225")]; - tensor inputs_85 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_225)[name = tensor("inputs_85")]; - tensor input_227 = silu(x = inputs_85)[name = tensor("input_227")]; - tensor input_229_pad_type_0 = const()[name = tensor("input_229_pad_type_0"), val = tensor("valid")]; - tensor input_229_strides_0 = const()[name = tensor("input_229_strides_0"), val = tensor([1])]; - tensor input_229_pad_0 = const()[name = tensor("input_229_pad_0"), val = tensor([0, 0])]; - tensor input_229_dilations_0 = const()[name = tensor("input_229_dilations_0"), val = tensor([1])]; - tensor input_229_groups_0 = const()[name = tensor("input_229_groups_0"), val = tensor(1)]; - tensor input_229 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_229_dilations_0, groups = input_229_groups_0, pad = input_229_pad_0, pad_type = input_229_pad_type_0, strides = input_229_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_227)[name = tensor("input_229")]; - tensor conv_out_23_perm_0 = const()[name = tensor("conv_out_23_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1619_begin_0 = const()[name = tensor("op_1619_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1619_end_0 = const()[name = tensor("op_1619_end_0"), val = tensor([1, 16, 256])]; - tensor var_1619_end_mask_0 = const()[name = tensor("op_1619_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_23 = transpose(perm = conv_out_23_perm_0, x = input_229)[name = tensor("transpose_31")]; - tensor var_1619 = slice_by_index(begin = var_1619_begin_0, end = var_1619_end_0, end_mask = var_1619_end_mask_0, x = conv_out_23)[name = tensor("op_1619")]; - tensor var_1622_interleave_0 = const()[name = tensor("op_1622_interleave_0"), val = tensor(false)]; - tensor var_1622 = concat(axis = var_33, interleave = var_1622_interleave_0, values = (var_1451, var_1535, var_1619))[name = tensor("op_1622")]; - tensor input_231 = add(x = x_33, y = var_1622)[name = tensor("input_231")]; - tensor input_233_axes_0 = const()[name = tensor("input_233_axes_0"), val = tensor([-1])]; - tensor input_233 = layer_norm(axes = input_233_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_231)[name = tensor("input_233")]; - tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_233)[name = tensor("linear_35")]; - tensor input_235 = silu(x = inputs)[name = tensor("input_235")]; - tensor input_239 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_235)[name = tensor("linear_36")]; - tensor var_1645 = const()[name = tensor("op_1645"), val = tensor(0x1p-1)]; - tensor var_1646 = mul(x = input_239, y = var_1645)[name = tensor("op_1646")]; - tensor input_241 = add(x = var_1646, y = input_231)[name = tensor("input_241")]; - tensor x_41_axes_0 = const()[name = tensor("x_41_axes_0"), val = tensor([-1])]; - tensor x_41 = layer_norm(axes = x_41_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_241)[name = tensor("x_41")]; + tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_35)[name = tensor("outputs_aug")]; + tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; + tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([3, 256, 16])]; + tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; + tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug)[name = tensor("input_145")]; + tensor inputs_37 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_145)[name = tensor("inputs_37")]; + tensor input_147 = silu(x = inputs_37)[name = tensor("input_147")]; + tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; + tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; + tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; + tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; + tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; + tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; + tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1192_begin_0 = const()[name = tensor("op_1192_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1192_end_0 = const()[name = tensor("op_1192_end_0"), val = tensor([3, 16, 256])]; + tensor var_1192_end_mask_0 = const()[name = tensor("op_1192_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_1192 = slice_by_index(begin = var_1192_begin_0, end = var_1192_end_0, end_mask = var_1192_end_mask_0, x = conv_out_7)[name = tensor("op_1192")]; + tensor var_1194_perm_0 = const()[name = tensor("op_1194_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1194 = transpose(perm = var_1194_perm_0, x = var_1192)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_1194)[name = tensor("input_151")]; + tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; + tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; + tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; + tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; + tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; + tensor var_1217 = const()[name = tensor("op_1217"), val = tensor(0x1p-1)]; + tensor var_1218 = mul(x = input_159, y = var_1217)[name = tensor("op_1218")]; + tensor input_161 = add(x = var_1218, y = input_151)[name = tensor("input_161")]; + tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; + tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; tensor cat_interleave_0 = const()[name = tensor("cat_interleave_0"), val = tensor(false)]; - tensor x_ct = transpose(perm = x_ct_perm_0, x = x_41)[name = tensor("transpose_30")]; + tensor x_ct = transpose(perm = x_ct_perm_0, x = x_25)[name = tensor("transpose_30")]; tensor cat = concat(axis = var_27, interleave = cat_interleave_0, values = (cnn_window, x_ct))[name = tensor("cat")]; tensor conv_out_pad_type_0 = const()[name = tensor("conv_out_pad_type_0"), val = tensor("valid")]; tensor conv_out_strides_0 = const()[name = tensor("conv_out_strides_0"), val = tensor([1])]; @@ -1513,44 +1210,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_1664_begin_0 = const()[name = tensor("op_1664_begin_0"), val = tensor([0, 0, 3])]; - tensor var_1664_end_0 = const()[name = tensor("op_1664_end_0"), val = tensor([1, 256, 21])]; - tensor var_1664_end_mask_0 = const()[name = tensor("op_1664_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_1664_begin_0, end = var_1664_end_0, end_mask = var_1664_end_mask_0, x = cat)[name = tensor("op_1664")]; - tensor input_243_perm_0 = const()[name = tensor("input_243_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1666 = const()[name = tensor("op_1666"), val = tensor([-1])]; - tensor input_243 = transpose(perm = input_243_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_1667 = reduce_l2_norm(axes = var_1666, keep_dims = var_36, x = input_243)[name = tensor("op_1667")]; - tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; - tensor clip_24 = clip(alpha = var_12, beta = const_20, x = var_1667)[name = tensor("clip_24")]; - tensor emb = real_div(x = input_243, y = clip_24)[name = tensor("emb")]; - tensor var_1671_axis_0 = const()[name = tensor("op_1671_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_1671_axis_0, values = (var_214, var_594, var_974, nkv_1))[name = tensor("op_1671")]; - tensor var_1673_axis_0 = const()[name = tensor("op_1673_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_1673_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1673")]; - tensor var_1675_axis_0 = const()[name = tensor("op_1675_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_1675_axis_0, values = (input_41, input_101, input_161, input_221))[name = tensor("op_1675")]; - tensor var_1677_axis_0 = const()[name = tensor("op_1677_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_1677_axis_0, values = (clip_5, clip_11, clip_17, clip_23))[name = tensor("op_1677")]; - tensor var_1687 = const()[name = tensor("op_1687"), val = tensor(0x1.5798eep-27)]; - tensor var_1693 = const()[name = tensor("op_1693"), val = tensor(0x1.4f8b58p-17)]; - tensor var_1695 = const()[name = tensor("op_1695"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_1696 = const()[name = tensor("op_1696"), val = tensor(true)]; - tensor var_1698 = const()[name = tensor("op_1698"), val = tensor(0x1p+0)]; - tensor var_1702 = const()[name = tensor("op_1702"), val = tensor(-1)]; - tensor var_1707 = const()[name = tensor("op_1707"), val = tensor(0)]; + tensor var_1236_begin_0 = const()[name = tensor("op_1236_begin_0"), val = tensor([0, 0, 3])]; + tensor var_1236_end_0 = const()[name = tensor("op_1236_end_0"), val = tensor([1, 256, 21])]; + tensor var_1236_end_mask_0 = const()[name = tensor("op_1236_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_1236_begin_0, end = var_1236_end_0, end_mask = var_1236_end_mask_0, x = cat)[name = tensor("op_1236")]; + tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1238 = const()[name = tensor("op_1238"), val = tensor([-1])]; + tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; + tensor var_1239 = reduce_l2_norm(axes = var_1238, keep_dims = var_36, x = input_163)[name = tensor("op_1239")]; + tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; + tensor clip_24 = clip(alpha = var_12, beta = const_12, x = var_1239)[name = tensor("clip_24")]; + tensor emb = real_div(x = input_163, y = clip_24)[name = tensor("emb")]; + tensor var_1243_axis_0 = const()[name = tensor("op_1243_axis_0"), val = tensor(0)]; + tensor enc_kv_new = stack(axis = var_1243_axis_0, values = (var_214, var_487, var_760, nkv_1))[name = tensor("op_1243")]; + tensor var_1245_axis_0 = const()[name = tensor("op_1245_axis_0"), val = tensor(0)]; + tensor enc_scale_new = stack(axis = var_1245_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1245")]; + tensor var_1247_axis_0 = const()[name = tensor("op_1247_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1247_axis_0, values = (window_7, window_15, window_23, window))[name = tensor("op_1247")]; + tensor var_1249_axis_0 = const()[name = tensor("op_1249_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1249_axis_0, values = (clip_5, clip_11, clip_17, clip_23))[name = tensor("op_1249")]; + tensor var_1259 = const()[name = tensor("op_1259"), val = tensor(0x1.5798eep-27)]; + tensor var_1264 = const()[name = tensor("op_1264"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1266 = const()[name = tensor("op_1266"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1267 = const()[name = tensor("op_1267"), val = tensor(true)]; + tensor var_1269 = const()[name = tensor("op_1269"), val = tensor(0x1p+0)]; + tensor var_1273 = const()[name = tensor("op_1273"), val = tensor(-1)]; + tensor var_1279 = const()[name = tensor("op_1279"), val = tensor(0)]; tensor pos = const()[name = tensor("pos"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395456)))]; - tensor var_1769_axes_0 = const()[name = tensor("op_1769_axes_0"), val = tensor([2])]; - tensor var_1769 = expand_dims(axes = var_1769_axes_0, x = emb)[name = tensor("op_1769")]; - tensor var_1770 = const()[name = tensor("op_1770"), val = tensor([1, 1, 6, 1])]; - tensor emb_exp = tile(reps = var_1770, x = var_1769)[name = tensor("emb_exp")]; - tensor input_245_interleave_0 = const()[name = tensor("input_245_interleave_0"), val = tensor(false)]; - tensor input_245 = concat(axis = var_1702, interleave = input_245_interleave_0, values = (emb_exp, pos))[name = tensor("input_245")]; - tensor x_43 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_245)[name = tensor("linear_37")]; - tensor var_1777_perm_0 = const()[name = tensor("op_1777_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1781 = const()[name = tensor("op_1781"), val = tensor([6, 3, 256])]; - tensor var_1777 = transpose(perm = var_1777_perm_0, x = x_43)[name = tensor("transpose_28")]; - tensor x_45 = reshape(shape = var_1781, x = var_1777)[name = tensor("x_45")]; + tensor var_1341_axes_0 = const()[name = tensor("op_1341_axes_0"), val = tensor([2])]; + tensor var_1341 = expand_dims(axes = var_1341_axes_0, x = emb)[name = tensor("op_1341")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 6, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1341)[name = tensor("emb_exp")]; + tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; + tensor input_165 = concat(axis = var_1273, interleave = input_165_interleave_0, values = (emb_exp, pos))[name = tensor("input_165")]; + tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; + tensor var_1349_perm_0 = const()[name = tensor("op_1349_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1353 = const()[name = tensor("op_1353"), val = tensor([6, 3, 256])]; + tensor var_1349 = transpose(perm = var_1349_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1353, x = var_1349)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 6, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1561,132 +1258,132 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_1789 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_45)[name = tensor("linear_38")]; - tensor var_1790 = const()[name = tensor("op_1790"), val = tensor([6, 3, 4, 64])]; - tensor var_1791 = reshape(shape = var_1790, x = var_1789)[name = tensor("op_1791")]; + tensor var_1361 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1362 = const()[name = tensor("op_1362"), val = tensor([6, 3, 4, 64])]; + tensor var_1363 = reshape(shape = var_1362, x = var_1361)[name = tensor("op_1363")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1795 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_45)[name = tensor("linear_39")]; - tensor var_1796 = const()[name = tensor("op_1796"), val = tensor(0x1p-3)]; - tensor var_1797 = mul(x = var_1795, y = var_1796)[name = tensor("op_1797")]; - tensor var_1798 = const()[name = tensor("op_1798"), val = tensor([6, 3, 4, 64])]; - tensor var_1799 = reshape(shape = var_1798, x = var_1797)[name = tensor("op_1799")]; + tensor var_1367 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1368 = const()[name = tensor("op_1368"), val = tensor(0x1p-3)]; + tensor var_1369 = mul(x = var_1367, y = var_1368)[name = tensor("op_1369")]; + tensor var_1370 = const()[name = tensor("op_1370"), val = tensor([6, 3, 4, 64])]; + tensor var_1371 = reshape(shape = var_1370, x = var_1369)[name = tensor("op_1371")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1803 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_45)[name = tensor("linear_40")]; - tensor var_1804 = const()[name = tensor("op_1804"), val = tensor([6, 3, 4, 64])]; - tensor var_1805 = reshape(shape = var_1804, x = var_1803)[name = tensor("op_1805")]; + tensor var_1375 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1376 = const()[name = tensor("op_1376"), val = tensor([6, 3, 4, 64])]; + tensor var_1377 = reshape(shape = var_1376, x = var_1375)[name = tensor("op_1377")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_249 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_45)[name = tensor("linear_41")]; + tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_1707, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1279, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; - tensor const_28 = const()[name = tensor("const_28"), val = tensor(0x1.fffffep+127)]; - tensor clip_25 = clip(alpha = var_1698, beta = const_28, x = s_t_9)[name = tensor("clip_25")]; + tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; + tensor clip_25 = clip(alpha = var_1269, beta = const_20, x = s_t_9)[name = tensor("clip_25")]; tensor sqrt_s_t_9 = sqrt(x = clip_25)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_1799)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_1791)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1371)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1363)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_1817 = const()[name = tensor("op_1817"), val = tensor([1, 3])]; - tensor var_1818 = reshape(shape = var_1817, x = valid_mask)[name = tensor("op_1818")]; - tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1818)[name = tensor("causal_with_valid_1")]; - tensor var_1820 = const()[name = tensor("op_1820"), val = tensor([3, 1])]; - tensor var_1821 = reshape(shape = var_1820, x = sqrt_s_t_9)[name = tensor("op_1821")]; - tensor M_9 = real_div(x = causal_with_valid_1, y = var_1821)[name = tensor("M_9")]; - tensor var_1823 = mul(x = qk_9, y = M_9)[name = tensor("op_1823")]; + tensor var_1389 = const()[name = tensor("op_1389"), val = tensor([1, 3])]; + tensor var_1390 = reshape(shape = var_1389, x = valid_mask)[name = tensor("op_1390")]; + tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1390)[name = tensor("causal_with_valid_1")]; + tensor var_1392 = const()[name = tensor("op_1392"), val = tensor([3, 1])]; + tensor var_1393 = reshape(shape = var_1392, x = sqrt_s_t_9)[name = tensor("op_1393")]; + tensor M_9 = real_div(x = causal_with_valid_1, y = var_1393)[name = tensor("M_9")]; + tensor var_1395 = mul(x = qk_9, y = M_9)[name = tensor("op_1395")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_1805)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1823, y = v_9)[name = tensor("inner_9")]; - tensor var_1825_transpose_x_0 = const()[name = tensor("op_1825_transpose_x_0"), val = tensor(false)]; - tensor var_1825_transpose_y_0 = const()[name = tensor("op_1825_transpose_y_0"), val = tensor(false)]; - tensor var_1825 = matmul(transpose_x = var_1825_transpose_x_0, transpose_y = var_1825_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1825")]; - tensor var_1826 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1826")]; - tensor var_1827 = const()[name = tensor("op_1827"), val = tensor([1, 1, 3, 1])]; - tensor var_1828 = reshape(shape = var_1827, x = var_1826)[name = tensor("op_1828")]; - tensor cross_9 = mul(x = var_1825, y = var_1828)[name = tensor("cross_9")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1377)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1395, y = v_9)[name = tensor("inner_9")]; + tensor var_1397_transpose_x_0 = const()[name = tensor("op_1397_transpose_x_0"), val = tensor(false)]; + tensor var_1397_transpose_y_0 = const()[name = tensor("op_1397_transpose_y_0"), val = tensor(false)]; + tensor var_1397 = matmul(transpose_x = var_1397_transpose_x_0, transpose_y = var_1397_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1397")]; + tensor var_1398 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1398")]; + tensor var_1399 = const()[name = tensor("op_1399"), val = tensor([1, 1, 3, 1])]; + tensor var_1400 = reshape(shape = var_1399, x = var_1398)[name = tensor("op_1400")]; + tensor cross_9 = mul(x = var_1397, y = var_1400)[name = tensor("cross_9")]; tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; - tensor var_1831 = const()[name = tensor("op_1831"), val = tensor([1, 1, 3, 1])]; - tensor var_1832 = reshape(shape = var_1831, x = valid_mask)[name = tensor("op_1832")]; - tensor v_masked_1 = mul(x = v_9, y = var_1832)[name = tensor("v_masked_1")]; - tensor var_1834 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1834")]; - tensor var_1836_transpose_x_1 = const()[name = tensor("op_1836_transpose_x_1"), val = tensor(true)]; - tensor var_1836_transpose_y_1 = const()[name = tensor("op_1836_transpose_y_1"), val = tensor(false)]; - tensor var_1836 = matmul(transpose_x = var_1836_transpose_x_1, transpose_y = var_1836_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1836")]; - tensor new_kv_unnorm_9 = add(x = var_1834, y = var_1836)[name = tensor("new_kv_unnorm_9")]; - tensor var_1838_keep_dims_0 = const()[name = tensor("op_1838_keep_dims_0"), val = tensor(false)]; - tensor var_1838 = reduce_sum(keep_dims = var_1838_keep_dims_0, x = valid_mask)[name = tensor("op_1838")]; - tensor var_1839 = const()[name = tensor("op_1839"), val = tensor([1])]; - tensor var_1840 = reshape(shape = var_1839, x = var_1838)[name = tensor("op_1840")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_1840)[name = tensor("new_scale_9")]; - tensor const_29 = const()[name = tensor("const_29"), val = tensor(0x1.fffffep+127)]; - tensor clip_26 = clip(alpha = var_1698, beta = const_29, x = new_scale_9)[name = tensor("clip_26")]; + tensor var_1403 = const()[name = tensor("op_1403"), val = tensor([1, 1, 3, 1])]; + tensor var_1404 = reshape(shape = var_1403, x = valid_mask)[name = tensor("op_1404")]; + tensor v_masked_1 = mul(x = v_9, y = var_1404)[name = tensor("v_masked_1")]; + tensor var_1406 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1406")]; + tensor var_1408_transpose_x_1 = const()[name = tensor("op_1408_transpose_x_1"), val = tensor(true)]; + tensor var_1408_transpose_y_1 = const()[name = tensor("op_1408_transpose_y_1"), val = tensor(false)]; + tensor var_1408 = matmul(transpose_x = var_1408_transpose_x_1, transpose_y = var_1408_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1408")]; + tensor new_kv_unnorm_9 = add(x = var_1406, y = var_1408)[name = tensor("new_kv_unnorm_9")]; + tensor var_1410_keep_dims_0 = const()[name = tensor("op_1410_keep_dims_0"), val = tensor(false)]; + tensor var_1410 = reduce_sum(keep_dims = var_1410_keep_dims_0, x = valid_mask)[name = tensor("op_1410")]; + tensor var_1411 = const()[name = tensor("op_1411"), val = tensor([1])]; + tensor var_1412 = reshape(shape = var_1411, x = var_1410)[name = tensor("op_1412")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1412)[name = tensor("new_scale_9")]; + tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; + tensor clip_26 = clip(alpha = var_1269, beta = const_21, x = new_scale_9)[name = tensor("clip_26")]; tensor sqrt_new_scale_1 = sqrt(x = clip_26)[name = tensor("sqrt_new_scale_1")]; - tensor var_1844 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1844")]; - tensor var_1845_perm_0 = const()[name = tensor("op_1845_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1416 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1416")]; + tensor var_1417_perm_0 = const()[name = tensor("op_1417_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_1845 = transpose(perm = var_1845_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1695, x = var_1845)[name = tensor("out_27")]; - tensor var_1849 = const()[name = tensor("op_1849"), val = tensor([6, 3, 256])]; - tensor out_29 = reshape(shape = var_1849, x = out_27)[name = tensor("out_29")]; - tensor var_1851 = silu(x = input_249)[name = tensor("op_1851")]; - tensor input_251 = mul(x = var_1851, y = out_29)[name = tensor("input_251")]; - tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_251)[name = tensor("linear_42")]; - tensor input_253 = add(x = x_45, y = ret_out_9)[name = tensor("input_253")]; + tensor var_1417 = transpose(perm = var_1417_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1266, x = var_1417)[name = tensor("out_27")]; + tensor var_1421 = const()[name = tensor("op_1421"), val = tensor([6, 3, 256])]; + tensor out_29 = reshape(shape = var_1421, x = out_27)[name = tensor("out_29")]; + tensor var_1423 = silu(x = input_169)[name = tensor("op_1423")]; + tensor input_171 = mul(x = var_1423, y = out_29)[name = tensor("input_171")]; + tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; + tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1693, gamma = decoder_norm11_0_weight, x = input_253)[name = tensor("xt_1")]; - tensor var_1861 = const()[name = tensor("op_1861"), val = tensor([1, 6, 3, 256])]; - tensor var_1862 = reshape(shape = var_1861, x = xt_1)[name = tensor("op_1862")]; - tensor var_1863_perm_0 = const()[name = tensor("op_1863_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1866 = const()[name = tensor("op_1866"), val = tensor([3, 6, 256])]; - tensor var_1863 = transpose(perm = var_1863_perm_0, x = var_1862)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_1866, x = var_1863)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1264, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1433 = const()[name = tensor("op_1433"), val = tensor([1, 6, 3, 256])]; + tensor var_1434 = reshape(shape = var_1433, x = xt_1)[name = tensor("op_1434")]; + tensor var_1435_perm_0 = const()[name = tensor("op_1435_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1438 = const()[name = tensor("op_1438"), val = tensor([3, 6, 256])]; + tensor var_1435 = transpose(perm = var_1435_perm_0, x = var_1434)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1438, x = var_1435)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_1889 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_12 = const()[name = tensor("concat_12"), val = tensor([6, 3, 3, 256])]; - tensor var_1891 = reshape(shape = concat_12, x = var_1889)[name = tensor("op_1891")]; - tensor var_1892_axes_0 = const()[name = tensor("op_1892_axes_0"), val = tensor([0])]; - tensor var_1892 = expand_dims(axes = var_1892_axes_0, x = var_1891)[name = tensor("op_1892")]; - tensor var_1893_perm_0 = const()[name = tensor("op_1893_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1894_axes_0 = const()[name = tensor("op_1894_axes_0"), val = tensor([-2])]; - tensor var_1893 = transpose(perm = var_1893_perm_0, x = var_1892)[name = tensor("transpose_21")]; - tensor var_1894 = squeeze(axes = var_1894_axes_0, x = var_1893)[name = tensor("op_1894")]; + tensor var_1461 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_13 = const()[name = tensor("concat_13"), val = tensor([6, 3, 3, 256])]; + tensor var_1463 = reshape(shape = concat_13, x = var_1461)[name = tensor("op_1463")]; + tensor var_1464_axes_0 = const()[name = tensor("op_1464_axes_0"), val = tensor([0])]; + tensor var_1464 = expand_dims(axes = var_1464_axes_0, x = var_1463)[name = tensor("op_1464")]; + tensor var_1465_perm_0 = const()[name = tensor("op_1465_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1466_axes_0 = const()[name = tensor("op_1466_axes_0"), val = tensor([-2])]; + tensor var_1465 = transpose(perm = var_1465_perm_0, x = var_1464)[name = tensor("transpose_21")]; + tensor var_1466 = squeeze(axes = var_1466_axes_0, x = var_1465)[name = tensor("op_1466")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 6, 3, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1894)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1466)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 6, 3, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1894)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1466)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 6, 3, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1894)[name = tensor("v_11")]; - tensor var_1902 = const()[name = tensor("op_1902"), val = tensor([6, 12, 64])]; - tensor var_1903 = reshape(shape = var_1902, x = q_11)[name = tensor("op_1903")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1466)[name = tensor("v_11")]; + tensor var_1474 = const()[name = tensor("op_1474"), val = tensor([6, 12, 64])]; + tensor var_1475 = reshape(shape = var_1474, x = q_11)[name = tensor("op_1475")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1909 = const()[name = tensor("op_1909"), val = tensor([6, 12, 64])]; - tensor var_1910 = reshape(shape = var_1909, x = k_11)[name = tensor("op_1910")]; + tensor var_1481 = const()[name = tensor("op_1481"), val = tensor([6, 12, 64])]; + tensor var_1482 = reshape(shape = var_1481, x = k_11)[name = tensor("op_1482")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1916 = const()[name = tensor("op_1916"), val = tensor([6, 12, 64])]; - tensor var_1917 = reshape(shape = var_1916, x = v_11)[name = tensor("op_1917")]; + tensor var_1488 = const()[name = tensor("op_1488"), val = tensor([6, 12, 64])]; + tensor var_1489 = reshape(shape = var_1488, x = v_11)[name = tensor("op_1489")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1920 = const()[name = tensor("op_1920"), val = tensor([3, 4, 6, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_1903)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_1920, x = q_13)[name = tensor("q_15")]; - tensor var_1922 = const()[name = tensor("op_1922"), val = tensor([3, 4, 6, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_1910)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_1922, x = k_13)[name = tensor("k_15")]; - tensor var_1924 = const()[name = tensor("op_1924"), val = tensor([3, 4, 6, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_1917)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_1924, x = v_13)[name = tensor("v_15")]; + tensor var_1492 = const()[name = tensor("op_1492"), val = tensor([3, 4, 6, 64])]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1475)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1492, x = q_13)[name = tensor("q_15")]; + tensor var_1494 = const()[name = tensor("op_1494"), val = tensor([3, 4, 6, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1482)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1494, x = k_13)[name = tensor("k_15")]; + tensor var_1496 = const()[name = tensor("op_1496"), val = tensor([3, 4, 6, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1489)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1496, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -1697,30 +1394,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_1927 = const()[name = tensor("op_1927"), val = tensor([2, 0, 1, 3])]; - tensor var_1932 = const()[name = tensor("op_1932"), val = tensor([18, 256])]; - tensor var_1928 = transpose(perm = var_1927, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_1932, x = var_1928)[name = tensor("attn_output_3")]; + tensor var_1499 = const()[name = tensor("op_1499"), val = tensor([2, 0, 1, 3])]; + tensor var_1504 = const()[name = tensor("op_1504"), val = tensor([18, 256])]; + tensor var_1500 = transpose(perm = var_1499, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1504, x = var_1500)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_1936 = const()[name = tensor("op_1936"), val = tensor([6, 3, 256])]; - tensor attn_output_7 = reshape(shape = var_1936, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1508 = const()[name = tensor("op_1508"), val = tensor([6, 3, 256])]; + tensor attn_output_7 = reshape(shape = var_1508, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; - tensor input_255 = add(x = query_1, y = sa2_out_1)[name = tensor("input_255")]; - tensor input_257_axes_0 = const()[name = tensor("input_257_axes_0"), val = tensor([-1])]; - tensor input_257 = layer_norm(axes = input_257_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1693, gamma = decoder_norm21_0_weight, x = input_255)[name = tensor("input_257")]; - tensor input_259 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_257)[name = tensor("linear_45")]; - tensor input_261 = relu(x = input_259)[name = tensor("input_261")]; - tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_261)[name = tensor("linear_46")]; - tensor input_263 = add(x = input_257, y = ffn_out_1)[name = tensor("input_263")]; + tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; + tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1264, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; + tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; + tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; + tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1693, gamma = decoder_norm22_0_weight, x = input_263)[name = tensor("xt_3")]; - tensor var_1956 = const()[name = tensor("op_1956"), val = tensor([1, 3, 6, 256])]; - tensor x_47 = reshape(shape = var_1956, x = xt_3)[name = tensor("x_47")]; - tensor var_1958_perm_0 = const()[name = tensor("op_1958_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1962 = const()[name = tensor("op_1962"), val = tensor([6, 3, 256])]; - tensor var_1958 = transpose(perm = var_1958_perm_0, x = x_47)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_1962, x = var_1958)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1264, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1528 = const()[name = tensor("op_1528"), val = tensor([1, 3, 6, 256])]; + tensor x_31 = reshape(shape = var_1528, x = xt_3)[name = tensor("x_31")]; + tensor var_1530_perm_0 = const()[name = tensor("op_1530_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1534 = const()[name = tensor("op_1534"), val = tensor([6, 3, 256])]; + tensor var_1530 = transpose(perm = var_1530_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1534, x = var_1530)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 6, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1731,120 +1428,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_1970 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_1971 = const()[name = tensor("op_1971"), val = tensor([6, 3, 4, 64])]; - tensor var_1972 = reshape(shape = var_1971, x = var_1970)[name = tensor("op_1972")]; + tensor var_1542 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1543 = const()[name = tensor("op_1543"), val = tensor([6, 3, 4, 64])]; + tensor var_1544 = reshape(shape = var_1543, x = var_1542)[name = tensor("op_1544")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1976 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_1977 = const()[name = tensor("op_1977"), val = tensor(0x1p-3)]; - tensor var_1978 = mul(x = var_1976, y = var_1977)[name = tensor("op_1978")]; - tensor var_1979 = const()[name = tensor("op_1979"), val = tensor([6, 3, 4, 64])]; - tensor var_1980 = reshape(shape = var_1979, x = var_1978)[name = tensor("op_1980")]; + tensor var_1548 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1549 = const()[name = tensor("op_1549"), val = tensor(0x1p-3)]; + tensor var_1550 = mul(x = var_1548, y = var_1549)[name = tensor("op_1550")]; + tensor var_1551 = const()[name = tensor("op_1551"), val = tensor([6, 3, 4, 64])]; + tensor var_1552 = reshape(shape = var_1551, x = var_1550)[name = tensor("op_1552")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1984 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_1985 = const()[name = tensor("op_1985"), val = tensor([6, 3, 4, 64])]; - tensor var_1986 = reshape(shape = var_1985, x = var_1984)[name = tensor("op_1986")]; + tensor var_1556 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1557 = const()[name = tensor("op_1557"), val = tensor([6, 3, 4, 64])]; + tensor var_1558 = reshape(shape = var_1557, x = var_1556)[name = tensor("op_1558")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_267 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; + tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; - tensor const_40 = const()[name = tensor("const_40"), val = tensor(0x1.fffffep+127)]; - tensor clip_27 = clip(alpha = var_1698, beta = const_40, x = s_t)[name = tensor("clip_27")]; + tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; + tensor clip_27 = clip(alpha = var_1269, beta = const_32, x = s_t)[name = tensor("clip_27")]; tensor sqrt_s_t = sqrt(x = clip_27)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_1980)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_1972)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1552)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1544)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_2001 = const()[name = tensor("op_2001"), val = tensor([3, 1])]; - tensor var_2002 = reshape(shape = var_2001, x = sqrt_s_t)[name = tensor("op_2002")]; - tensor M = real_div(x = causal_with_valid_1, y = var_2002)[name = tensor("M")]; - tensor var_2004 = mul(x = qk, y = M)[name = tensor("op_2004")]; + tensor var_1573 = const()[name = tensor("op_1573"), val = tensor([3, 1])]; + tensor var_1574 = reshape(shape = var_1573, x = sqrt_s_t)[name = tensor("op_1574")]; + tensor M = real_div(x = causal_with_valid_1, y = var_1574)[name = tensor("M")]; + tensor var_1576 = mul(x = qk, y = M)[name = tensor("op_1576")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_1986)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_2004, y = v_17)[name = tensor("inner")]; - tensor var_2006_transpose_x_0 = const()[name = tensor("op_2006_transpose_x_0"), val = tensor(false)]; - tensor var_2006_transpose_y_0 = const()[name = tensor("op_2006_transpose_y_0"), val = tensor(false)]; - tensor var_2006 = matmul(transpose_x = var_2006_transpose_x_0, transpose_y = var_2006_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_2006")]; - tensor var_2007 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_2007")]; - tensor var_2008 = const()[name = tensor("op_2008"), val = tensor([1, 1, 3, 1])]; - tensor var_2009 = reshape(shape = var_2008, x = var_2007)[name = tensor("op_2009")]; - tensor cross = mul(x = var_2006, y = var_2009)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1558)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1576, y = v_17)[name = tensor("inner")]; + tensor var_1578_transpose_x_0 = const()[name = tensor("op_1578_transpose_x_0"), val = tensor(false)]; + tensor var_1578_transpose_y_0 = const()[name = tensor("op_1578_transpose_y_0"), val = tensor(false)]; + tensor var_1578 = matmul(transpose_x = var_1578_transpose_x_0, transpose_y = var_1578_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1578")]; + tensor var_1579 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1579")]; + tensor var_1580 = const()[name = tensor("op_1580"), val = tensor([1, 1, 3, 1])]; + tensor var_1581 = reshape(shape = var_1580, x = var_1579)[name = tensor("op_1581")]; + tensor cross = mul(x = var_1578, y = var_1581)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_1832)[name = tensor("v_masked")]; - tensor var_2015 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_2015")]; - tensor var_2017_transpose_x_1 = const()[name = tensor("op_2017_transpose_x_1"), val = tensor(true)]; - tensor var_2017_transpose_y_1 = const()[name = tensor("op_2017_transpose_y_1"), val = tensor(false)]; - tensor var_2017 = matmul(transpose_x = var_2017_transpose_x_1, transpose_y = var_2017_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_2017")]; - tensor new_kv_unnorm = add(x = var_2015, y = var_2017)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_1840)[name = tensor("new_scale")]; - tensor const_41 = const()[name = tensor("const_41"), val = tensor(0x1.fffffep+127)]; - tensor clip_28 = clip(alpha = var_1698, beta = const_41, x = new_scale)[name = tensor("clip_28")]; + tensor v_masked = mul(x = v_17, y = var_1404)[name = tensor("v_masked")]; + tensor var_1587 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1587")]; + tensor var_1589_transpose_x_1 = const()[name = tensor("op_1589_transpose_x_1"), val = tensor(true)]; + tensor var_1589_transpose_y_1 = const()[name = tensor("op_1589_transpose_y_1"), val = tensor(false)]; + tensor var_1589 = matmul(transpose_x = var_1589_transpose_x_1, transpose_y = var_1589_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1589")]; + tensor new_kv_unnorm = add(x = var_1587, y = var_1589)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1412)[name = tensor("new_scale")]; + tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; + tensor clip_28 = clip(alpha = var_1269, beta = const_33, x = new_scale)[name = tensor("clip_28")]; tensor sqrt_new_scale = sqrt(x = clip_28)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_2026_perm_0 = const()[name = tensor("op_2026_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1598_perm_0 = const()[name = tensor("op_1598_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_2026 = transpose(perm = var_2026_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1695, x = var_2026)[name = tensor("out_33")]; - tensor var_2030 = const()[name = tensor("op_2030"), val = tensor([6, 3, 256])]; - tensor out = reshape(shape = var_2030, x = out_33)[name = tensor("out")]; - tensor var_2032 = silu(x = input_267)[name = tensor("op_2032")]; - tensor input_269 = mul(x = var_2032, y = out)[name = tensor("input_269")]; - tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_269)[name = tensor("linear_51")]; - tensor input_271 = add(x = x, y = ret_out)[name = tensor("input_271")]; + tensor var_1598 = transpose(perm = var_1598_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1266, x = var_1598)[name = tensor("out_33")]; + tensor var_1602 = const()[name = tensor("op_1602"), val = tensor([6, 3, 256])]; + tensor out = reshape(shape = var_1602, x = out_33)[name = tensor("out")]; + tensor var_1604 = silu(x = input_187)[name = tensor("op_1604")]; + tensor input_189 = mul(x = var_1604, y = out)[name = tensor("input_189")]; + tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; + tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1693, gamma = decoder_norm11_1_weight, x = input_271)[name = tensor("xt_5")]; - tensor var_2042 = const()[name = tensor("op_2042"), val = tensor([1, 6, 3, 256])]; - tensor var_2043 = reshape(shape = var_2042, x = xt_5)[name = tensor("op_2043")]; - tensor var_2044_perm_0 = const()[name = tensor("op_2044_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2047 = const()[name = tensor("op_2047"), val = tensor([3, 6, 256])]; - tensor var_2044 = transpose(perm = var_2044_perm_0, x = var_2043)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_2047, x = var_2044)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1264, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1614 = const()[name = tensor("op_1614"), val = tensor([1, 6, 3, 256])]; + tensor var_1615 = reshape(shape = var_1614, x = xt_5)[name = tensor("op_1615")]; + tensor var_1616_perm_0 = const()[name = tensor("op_1616_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1619 = const()[name = tensor("op_1619"), val = tensor([3, 6, 256])]; + tensor var_1616 = transpose(perm = var_1616_perm_0, x = var_1615)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1619, x = var_1616)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_2070 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_13 = const()[name = tensor("concat_13"), val = tensor([6, 3, 3, 256])]; - tensor var_2072 = reshape(shape = concat_13, x = var_2070)[name = tensor("op_2072")]; - tensor var_2073_axes_0 = const()[name = tensor("op_2073_axes_0"), val = tensor([0])]; - tensor var_2073 = expand_dims(axes = var_2073_axes_0, x = var_2072)[name = tensor("op_2073")]; - tensor var_2074_perm_0 = const()[name = tensor("op_2074_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2075_axes_0 = const()[name = tensor("op_2075_axes_0"), val = tensor([-2])]; - tensor var_2074 = transpose(perm = var_2074_perm_0, x = var_2073)[name = tensor("transpose_8")]; - tensor var_2075 = squeeze(axes = var_2075_axes_0, x = var_2074)[name = tensor("op_2075")]; + tensor var_1642 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_14 = const()[name = tensor("concat_14"), val = tensor([6, 3, 3, 256])]; + tensor var_1644 = reshape(shape = concat_14, x = var_1642)[name = tensor("op_1644")]; + tensor var_1645_axes_0 = const()[name = tensor("op_1645_axes_0"), val = tensor([0])]; + tensor var_1645 = expand_dims(axes = var_1645_axes_0, x = var_1644)[name = tensor("op_1645")]; + tensor var_1646_perm_0 = const()[name = tensor("op_1646_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1647_axes_0 = const()[name = tensor("op_1647_axes_0"), val = tensor([-2])]; + tensor var_1646 = transpose(perm = var_1646_perm_0, x = var_1645)[name = tensor("transpose_8")]; + tensor var_1647 = squeeze(axes = var_1647_axes_0, x = var_1646)[name = tensor("op_1647")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 6, 3, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_2075)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1647)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 6, 3, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_2075)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1647)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 6, 3, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_2075)[name = tensor("v_19")]; - tensor var_2083 = const()[name = tensor("op_2083"), val = tensor([6, 12, 64])]; - tensor var_2084 = reshape(shape = var_2083, x = q_19)[name = tensor("op_2084")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1647)[name = tensor("v_19")]; + tensor var_1655 = const()[name = tensor("op_1655"), val = tensor([6, 12, 64])]; + tensor var_1656 = reshape(shape = var_1655, x = q_19)[name = tensor("op_1656")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2090 = const()[name = tensor("op_2090"), val = tensor([6, 12, 64])]; - tensor var_2091 = reshape(shape = var_2090, x = k_19)[name = tensor("op_2091")]; + tensor var_1662 = const()[name = tensor("op_1662"), val = tensor([6, 12, 64])]; + tensor var_1663 = reshape(shape = var_1662, x = k_19)[name = tensor("op_1663")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2097 = const()[name = tensor("op_2097"), val = tensor([6, 12, 64])]; - tensor var_2098 = reshape(shape = var_2097, x = v_19)[name = tensor("op_2098")]; + tensor var_1669 = const()[name = tensor("op_1669"), val = tensor([6, 12, 64])]; + tensor var_1670 = reshape(shape = var_1669, x = v_19)[name = tensor("op_1670")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2101 = const()[name = tensor("op_2101"), val = tensor([3, 4, 6, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_2084)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_2101, x = q_21)[name = tensor("q")]; - tensor var_2103 = const()[name = tensor("op_2103"), val = tensor([3, 4, 6, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_2091)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_2103, x = k_21)[name = tensor("k")]; - tensor var_2105 = const()[name = tensor("op_2105"), val = tensor([3, 4, 6, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_2098)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_2105, x = v_21)[name = tensor("v")]; + tensor var_1673 = const()[name = tensor("op_1673"), val = tensor([3, 4, 6, 64])]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1656)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1673, x = q_21)[name = tensor("q")]; + tensor var_1675 = const()[name = tensor("op_1675"), val = tensor([3, 4, 6, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1663)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1675, x = k_21)[name = tensor("k")]; + tensor var_1677 = const()[name = tensor("op_1677"), val = tensor([3, 4, 6, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1670)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1677, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -1855,50 +1552,50 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_2108 = const()[name = tensor("op_2108"), val = tensor([2, 0, 1, 3])]; - tensor var_2113 = const()[name = tensor("op_2113"), val = tensor([18, 256])]; - tensor var_2109 = transpose(perm = var_2108, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_2113, x = var_2109)[name = tensor("attn_output_11")]; + tensor var_1680 = const()[name = tensor("op_1680"), val = tensor([2, 0, 1, 3])]; + tensor var_1685 = const()[name = tensor("op_1685"), val = tensor([18, 256])]; + tensor var_1681 = transpose(perm = var_1680, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1685, x = var_1681)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_2117 = const()[name = tensor("op_2117"), val = tensor([6, 3, 256])]; - tensor attn_output = reshape(shape = var_2117, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1689 = const()[name = tensor("op_1689"), val = tensor([6, 3, 256])]; + tensor attn_output = reshape(shape = var_1689, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; - tensor input_273 = add(x = query_5, y = sa2_out)[name = tensor("input_273")]; - tensor input_275_axes_0 = const()[name = tensor("input_275_axes_0"), val = tensor([-1])]; - tensor input_275 = layer_norm(axes = input_275_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1693, gamma = decoder_norm21_1_weight, x = input_273)[name = tensor("input_275")]; - tensor input_277 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_275)[name = tensor("linear_54")]; - tensor input_279 = relu(x = input_277)[name = tensor("input_279")]; - tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_279)[name = tensor("linear_55")]; - tensor input_281 = add(x = input_275, y = ffn_out)[name = tensor("input_281")]; + tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; + tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1264, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; + tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; + tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; + tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1693, gamma = decoder_norm22_1_weight, x = input_281)[name = tensor("xt")]; - tensor var_2137 = const()[name = tensor("op_2137"), val = tensor([1, 3, 6, 256])]; - tensor input = reshape(shape = var_2137, x = xt)[name = tensor("input")]; - tensor var_2139 = const()[name = tensor("op_2139"), val = tensor([-1])]; - tensor var_2140 = reduce_l2_norm(axes = var_2139, keep_dims = var_1696, x = input)[name = tensor("op_2140")]; - tensor const_50 = const()[name = tensor("const_50"), val = tensor(0x1.fffffep+127)]; - tensor clip_29 = clip(alpha = var_1687, beta = const_50, x = var_2140)[name = tensor("clip_29")]; - tensor var_2142 = real_div(x = input, y = clip_29)[name = tensor("op_2142")]; - tensor concat_17 = const()[name = tensor("concat_17"), val = tensor([3, 1, 256])]; - tensor reshape_0 = reshape(shape = concat_17, x = emb)[name = tensor("reshape_0")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1264, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1709 = const()[name = tensor("op_1709"), val = tensor([1, 3, 6, 256])]; + tensor input = reshape(shape = var_1709, x = xt)[name = tensor("input")]; + tensor var_1711 = const()[name = tensor("op_1711"), val = tensor([-1])]; + tensor var_1712 = reduce_l2_norm(axes = var_1711, keep_dims = var_1267, x = input)[name = tensor("op_1712")]; + tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; + tensor clip_29 = clip(alpha = var_1259, beta = const_42, x = var_1712)[name = tensor("clip_29")]; + tensor var_1714 = real_div(x = input, y = clip_29)[name = tensor("op_1714")]; + tensor concat_18 = const()[name = tensor("concat_18"), val = tensor([3, 1, 256])]; + tensor reshape_0 = reshape(shape = concat_18, x = emb)[name = tensor("reshape_0")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_18 = const()[name = tensor("concat_18"), val = tensor([3, 256, 6])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_2142)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_18, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_19 = const()[name = tensor("concat_19"), val = tensor([3, 256, 6])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1714)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_19, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = reshape_0, y = reshape_1)[name = tensor("matmul_2")]; - tensor concat_21 = const()[name = tensor("concat_21"), val = tensor([1, 3, 6])]; - tensor reshape_2 = reshape(shape = concat_21, x = matmul_2)[name = tensor("reshape_2")]; + tensor concat_22 = const()[name = tensor("concat_22"), val = tensor([1, 3, 6])]; + tensor reshape_2 = reshape(shape = concat_22, x = matmul_2)[name = tensor("reshape_2")]; tensor output_begin_0 = const()[name = tensor("output_begin_0"), val = tensor([0, 0, 1])]; tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 3, 5])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = reshape_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_2146")]; - tensor var_2148_axis_0 = const()[name = tensor("op_2148_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_2148_axis_0, values = (var_1844, nkv))[name = tensor("op_2148")]; - tensor var_2150_axis_0 = const()[name = tensor("op_2150_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_2150_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_2150")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1718")]; + tensor var_1720_axis_0 = const()[name = tensor("op_1720_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1720_axis_0, values = (var_1416, nkv))[name = tensor("op_1720")]; + tensor var_1722_axis_0 = const()[name = tensor("op_1722_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1722_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1722")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/AMI/ls_eend_ami_300ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/AMI/ls_eend_ami_300ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index 855035e8f2f5dec4cdd8e58640cdfe60c3301562..097e2b95bce2a0a646ee22688ad6b24e2e382723 100644 --- a/optimized/AMI/ls_eend_ami_300ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/AMI/ls_eend_ami_300ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:65b017e5da5aeae5d4c4a7323837a05014986175c8f41d37770f2a64cc4b888a -size 269866 +oid sha256:a3425cc545be5da6480e1ef8593c801f8df6c5a3e93faaf2790a15af8daa9cc4 +size 222317 diff --git a/optimized/AMI/ls_eend_ami_300ms.mlpackage/Manifest.json b/optimized/AMI/ls_eend_ami_300ms.mlpackage/Manifest.json index 275e89443c66bbed7eb5fbc0ab0c2606af7e0486..617756f61637aaeca243a73bae26318cfd7acb27 100644 --- a/optimized/AMI/ls_eend_ami_300ms.mlpackage/Manifest.json +++ b/optimized/AMI/ls_eend_ami_300ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "233434E2-9E6B-4402-8E5D-F5C9D62B4FFD": { + "25895652-3706-4319-97D9-B8C6E0C3C940": { "author": "com.apple.CoreML", "description": "CoreML Model Specification", "name": "model.mlmodel", "path": "com.apple.CoreML/model.mlmodel" }, - "4C7E7012-D3B0-479A-AC38-6EF2E5D3591A": { + "2A899BED-6F3E-444E-A37E-2D1568DC3DB0": { "author": "com.apple.CoreML", "description": "CoreML Model Weights", "name": "weights", "path": "com.apple.CoreML/weights" } }, - "rootModelIdentifier": "233434E2-9E6B-4402-8E5D-F5C9D62B4FFD" + "rootModelIdentifier": "25895652-3706-4319-97D9-B8C6E0C3C940" } diff --git a/optimized/AMI/ls_eend_ami_400ms.mlmodelc/analytics/coremldata.bin b/optimized/AMI/ls_eend_ami_400ms.mlmodelc/analytics/coremldata.bin index 0d691d949b634b0bb67431b0af74bcfc4df34032..a020d25fbdd9fe8b586d5d7e247be7c7b92e9eb9 100644 --- a/optimized/AMI/ls_eend_ami_400ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/AMI/ls_eend_ami_400ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2e9c0efdb1afd4e2d3654c7191792fff76ebbe05f95a2c31f1bc231966434560 +oid sha256:f7fbd29536b5827cc4bb17e7881ce9c8ece324f279e517978d99c7b69fda25ab size 243 diff --git a/optimized/AMI/ls_eend_ami_400ms.mlmodelc/coremldata.bin b/optimized/AMI/ls_eend_ami_400ms.mlmodelc/coremldata.bin index 7db04fcb2de8a70a62b88644bdeb93fdfb895de1..980cd33d22f3d17ad7d44c66b2fb9efaac43c635 100644 --- a/optimized/AMI/ls_eend_ami_400ms.mlmodelc/coremldata.bin +++ b/optimized/AMI/ls_eend_ami_400ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:364e00f3f7a4792117cd7b6b32de53e5d36cf672a5f0076326ac0dc7742e18d5 -size 1250 +oid sha256:1209f9623fb002d16dc2a2d46a297f51dc5b931ae0f055244e2ea4412312ded5 +size 1341 diff --git a/optimized/AMI/ls_eend_ami_400ms.mlmodelc/metadata.json b/optimized/AMI/ls_eend_ami_400ms.mlmodelc/metadata.json index a2a312f3e2f6a4b661e8bc06e5cc3b7608ea20ec..34eb757536e63fec013de5eb7197520972250e8a 100644 --- a/optimized/AMI/ls_eend_ami_400ms.mlmodelc/metadata.json +++ b/optimized/AMI/ls_eend_ami_400ms.mlmodelc/metadata.json @@ -96,30 +96,30 @@ "Ios17.reshape" : 100, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 77, - "Split" : 16, + "Ios17.transpose" : 57, + "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 94, "Tile" : 17, - "Ios17.sliceByIndex" : 92, - "Ios16.sigmoid" : 17, + "Ios17.sliceByIndex" : 68, + "Ios16.sigmoid" : 5, "Ios16.reduceSum" : 1, "Ios17.squeeze" : 2, - "Ios17.layerNorm" : 45, - "Ios17.batchNorm" : 16, + "Ios17.layerNorm" : 33, + "Ios17.batchNorm" : 4, "Ios17.cast" : 32, "Ios17.sqrt" : 18, "Ios17.sub" : 32, - "Ios17.conv" : 49, + "Ios17.conv" : 13, "Ios17.realDiv" : 20, "Ios17.linear" : 56, "Ios17.concat" : 22, "Ios17.clip" : 38, "Ios17.greaterEqual" : 16, - "Ios16.silu" : 30, + "Ios16.silu" : 18, "Stack" : 6, "Ios16.cumsum" : 1, - "Ios17.mul" : 123 + "Ios17.mul" : 111 }, "computePrecision" : "Mixed (Float32, Int32)", "isUpdatable" : "0", @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"ami\", \"model_label\": \"AMI\", \"variant\": \"pipeline\", \"T\": 4, \"max_speakers\": 4, \"max_nspks\": 6, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"ami\", \"model_label\": \"AMI\", \"variant\": \"pipeline\", \"chunk_size\": 4, \"step_duration_ms\": 400, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 4, \"max_nspks\": 6, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/AMI/ls_eend_ami_400ms.mlmodelc/model.mil b/optimized/AMI/ls_eend_ami_400ms.mlmodelc/model.mil index 77de5cdf09c7de4460023002fce44e2b58f5c2be..5b5e14a94060bcdd6a7e00ec62b847390809f488 100644 --- a/optimized/AMI/ls_eend_ami_400ms.mlmodelc/model.mil +++ b/optimized/AMI/ls_eend_ami_400ms.mlmodelc/model.mil @@ -217,6 +217,7 @@ program(1.0) tensor var_21 = const()[name = tensor("op_21"), val = tensor(0x0p+0)]; tensor var_24 = const()[name = tensor("op_24"), val = tensor(0x1.0c6f7ap-20)]; tensor var_27 = const()[name = tensor("op_27"), val = tensor(2)]; + tensor var_30 = const()[name = tensor("op_30"), val = tensor(0)]; tensor var_33 = const()[name = tensor("op_33"), val = tensor(1)]; tensor var_35 = const()[name = tensor("op_35"), val = tensor(0x1.4f8b58p-17)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(true)]; @@ -263,8 +264,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_77")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_78")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_196 = const()[name = tensor("op_196"), val = tensor([4, 1])]; tensor var_197 = reshape(shape = var_196, x = sqrt_s_t_1)[name = tensor("op_197")]; @@ -272,7 +273,7 @@ program(1.0) tensor var_199 = mul(x = qk_1, y = M_1)[name = tensor("op_199")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_76")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_199, y = v_1)[name = tensor("inner_1")]; tensor var_201_transpose_x_0 = const()[name = tensor("op_201_transpose_x_0"), val = tensor(false)]; tensor var_201_transpose_y_0 = const()[name = tensor("op_201_transpose_y_0"), val = tensor(false)]; @@ -293,7 +294,7 @@ program(1.0) tensor var_214 = real_div(x = new_kv_unnorm_1, y = var_213)[name = tensor("op_214")]; tensor var_215_perm_0 = const()[name = tensor("op_215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_75")]; + tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_24, x = var_215)[name = tensor("out_3")]; tensor var_219 = const()[name = tensor("op_219"), val = tensor([1, 4, 256])]; tensor out_5 = reshape(shape = var_219, x = out_3)[name = tensor("out_5")]; @@ -341,291 +342,176 @@ program(1.0) tensor var_252 = sub(x = var_17, y = use_roll_1)[name = tensor("op_252")]; tensor var_253 = mul(x = filled_window_1, y = var_252)[name = tensor("op_253")]; tensor var_254 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_254")]; - tensor input_21 = add(x = var_253, y = var_254)[name = tensor("input_21")]; + tensor window_3 = add(x = var_253, y = var_254)[name = tensor("window_3")]; tensor var_256 = const()[name = tensor("op_256"), val = tensor(0x1p+0)]; tensor var_257 = add(x = filled_1, y = var_256)[name = tensor("op_257")]; tensor clip_1 = clip(alpha = var_21, beta = var_16, x = var_257)[name = tensor("clip_1")]; + tensor var_260_begin_0 = const()[name = tensor("op_260_begin_0"), val = tensor([0, 1, 0])]; + tensor var_260_end_0 = const()[name = tensor("op_260_end_0"), val = tensor([1, 2, 256])]; + tensor var_260_end_mask_0 = const()[name = tensor("op_260_end_mask_0"), val = tensor([true, false, true])]; + tensor var_260 = slice_by_index(begin = var_260_begin_0, end = var_260_end_0, end_mask = var_260_end_mask_0, x = x_3)[name = tensor("op_260")]; + tensor var_263_begin_0 = const()[name = tensor("op_263_begin_0"), val = tensor([0, 1, 0])]; + tensor var_263_end_0 = const()[name = tensor("op_263_end_0"), val = tensor([1, 16, 256])]; + tensor var_263_end_mask_0 = const()[name = tensor("op_263_end_mask_0"), val = tensor([true, true, true])]; + tensor var_263 = slice_by_index(begin = var_263_begin_0, end = var_263_end_0, end_mask = var_263_end_mask_0, x = window_3)[name = tensor("op_263")]; + tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; + tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_263, var_260))[name = tensor("rolled_3")]; + tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; + tensor var_268 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_268")]; + tensor var_269_dtype_0 = const()[name = tensor("op_269_dtype_0"), val = tensor("fp32")]; + tensor var_270 = const()[name = tensor("op_270"), val = tensor([1, 16, 1])]; + tensor var_269 = cast(dtype = var_269_dtype_0, x = var_268)[name = tensor("cast_173")]; + tensor fill_mask_3 = reshape(shape = var_270, x = var_269)[name = tensor("fill_mask_3")]; + tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_260)[name = tensor("x_expanded_3")]; + tensor var_274 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_274")]; + tensor var_275 = mul(x = window_3, y = var_274)[name = tensor("op_275")]; + tensor var_276 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_276")]; + tensor filled_window_3 = add(x = var_275, y = var_276)[name = tensor("filled_window_3")]; + tensor var_278 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_278")]; + tensor var_279_dtype_0 = const()[name = tensor("op_279_dtype_0"), val = tensor("fp32")]; + tensor var_280 = const()[name = tensor("op_280"), val = tensor([1, 1, 1])]; + tensor var_279 = cast(dtype = var_279_dtype_0, x = var_278)[name = tensor("cast_172")]; + tensor use_roll_3 = reshape(shape = var_280, x = var_279)[name = tensor("use_roll_3")]; + tensor var_282 = sub(x = var_17, y = use_roll_3)[name = tensor("op_282")]; + tensor var_283 = mul(x = filled_window_3, y = var_282)[name = tensor("op_283")]; + tensor var_284 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_284")]; + tensor window_5 = add(x = var_283, y = var_284)[name = tensor("window_5")]; + tensor var_286 = const()[name = tensor("op_286"), val = tensor(0x1p+0)]; + tensor var_287 = add(x = clip_1, y = var_286)[name = tensor("op_287")]; + tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_287)[name = tensor("clip_3")]; + tensor var_290_begin_0 = const()[name = tensor("op_290_begin_0"), val = tensor([0, 2, 0])]; + tensor var_290_end_0 = const()[name = tensor("op_290_end_0"), val = tensor([1, 3, 256])]; + tensor var_290_end_mask_0 = const()[name = tensor("op_290_end_mask_0"), val = tensor([true, false, true])]; + tensor var_290 = slice_by_index(begin = var_290_begin_0, end = var_290_end_0, end_mask = var_290_end_mask_0, x = x_3)[name = tensor("op_290")]; + tensor var_293_begin_0 = const()[name = tensor("op_293_begin_0"), val = tensor([0, 1, 0])]; + tensor var_293_end_0 = const()[name = tensor("op_293_end_0"), val = tensor([1, 16, 256])]; + tensor var_293_end_mask_0 = const()[name = tensor("op_293_end_mask_0"), val = tensor([true, true, true])]; + tensor var_293 = slice_by_index(begin = var_293_begin_0, end = var_293_end_0, end_mask = var_293_end_mask_0, x = window_5)[name = tensor("op_293")]; + tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; + tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_293, var_290))[name = tensor("rolled_5")]; + tensor clip_4 = clip(alpha = var_21, beta = var_20, x = clip_3)[name = tensor("clip_4")]; + tensor var_298 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_298")]; + tensor var_299_dtype_0 = const()[name = tensor("op_299_dtype_0"), val = tensor("fp32")]; + tensor var_300 = const()[name = tensor("op_300"), val = tensor([1, 16, 1])]; + tensor var_299 = cast(dtype = var_299_dtype_0, x = var_298)[name = tensor("cast_171")]; + tensor fill_mask_5 = reshape(shape = var_300, x = var_299)[name = tensor("fill_mask_5")]; + tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_290)[name = tensor("x_expanded_5")]; + tensor var_304 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_304")]; + tensor var_305 = mul(x = window_5, y = var_304)[name = tensor("op_305")]; + tensor var_306 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_306")]; + tensor filled_window_5 = add(x = var_305, y = var_306)[name = tensor("filled_window_5")]; + tensor var_308 = greater_equal(x = clip_3, y = var_16)[name = tensor("op_308")]; + tensor var_309_dtype_0 = const()[name = tensor("op_309_dtype_0"), val = tensor("fp32")]; + tensor var_310 = const()[name = tensor("op_310"), val = tensor([1, 1, 1])]; + tensor var_309 = cast(dtype = var_309_dtype_0, x = var_308)[name = tensor("cast_170")]; + tensor use_roll_5 = reshape(shape = var_310, x = var_309)[name = tensor("use_roll_5")]; + tensor var_312 = sub(x = var_17, y = use_roll_5)[name = tensor("op_312")]; + tensor var_313 = mul(x = filled_window_5, y = var_312)[name = tensor("op_313")]; + tensor var_314 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_314")]; + tensor window_7 = add(x = var_313, y = var_314)[name = tensor("window_7")]; + tensor var_316 = const()[name = tensor("op_316"), val = tensor(0x1p+0)]; + tensor var_317 = add(x = clip_3, y = var_316)[name = tensor("op_317")]; + tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_317)[name = tensor("clip_5")]; + tensor var_320_begin_0 = const()[name = tensor("op_320_begin_0"), val = tensor([0, 3, 0])]; + tensor var_320_end_0 = const()[name = tensor("op_320_end_0"), val = tensor([1, 1, 256])]; + tensor var_320_end_mask_0 = const()[name = tensor("op_320_end_mask_0"), val = tensor([true, true, true])]; + tensor var_320 = slice_by_index(begin = var_320_begin_0, end = var_320_end_0, end_mask = var_320_end_mask_0, x = x_3)[name = tensor("op_320")]; + tensor var_323_begin_0 = const()[name = tensor("op_323_begin_0"), val = tensor([0, 1, 0])]; + tensor var_323_end_0 = const()[name = tensor("op_323_end_0"), val = tensor([1, 16, 256])]; + tensor var_323_end_mask_0 = const()[name = tensor("op_323_end_mask_0"), val = tensor([true, true, true])]; + tensor var_323 = slice_by_index(begin = var_323_begin_0, end = var_323_end_0, end_mask = var_323_end_mask_0, x = window_7)[name = tensor("op_323")]; + tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; + tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_323, var_320))[name = tensor("rolled_7")]; + tensor clip_6 = clip(alpha = var_21, beta = var_20, x = clip_5)[name = tensor("clip_6")]; + tensor var_328 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_328")]; + tensor var_329_dtype_0 = const()[name = tensor("op_329_dtype_0"), val = tensor("fp32")]; + tensor var_330 = const()[name = tensor("op_330"), val = tensor([1, 16, 1])]; + tensor var_329 = cast(dtype = var_329_dtype_0, x = var_328)[name = tensor("cast_169")]; + tensor fill_mask_7 = reshape(shape = var_330, x = var_329)[name = tensor("fill_mask_7")]; + tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_320)[name = tensor("x_expanded_7")]; + tensor var_334 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_334")]; + tensor var_335 = mul(x = window_7, y = var_334)[name = tensor("op_335")]; + tensor var_336 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_336")]; + tensor filled_window_7 = add(x = var_335, y = var_336)[name = tensor("filled_window_7")]; + tensor var_338 = greater_equal(x = clip_5, y = var_16)[name = tensor("op_338")]; + tensor var_339_dtype_0 = const()[name = tensor("op_339_dtype_0"), val = tensor("fp32")]; + tensor var_340 = const()[name = tensor("op_340"), val = tensor([1, 1, 1])]; + tensor var_339 = cast(dtype = var_339_dtype_0, x = var_338)[name = tensor("cast_168")]; + tensor use_roll_7 = reshape(shape = var_340, x = var_339)[name = tensor("use_roll_7")]; + tensor var_342 = sub(x = var_17, y = use_roll_7)[name = tensor("op_342")]; + tensor var_343 = mul(x = filled_window_7, y = var_342)[name = tensor("op_343")]; + tensor var_344 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_344")]; + tensor window_9 = add(x = var_343, y = var_344)[name = tensor("window_9")]; + tensor var_346 = const()[name = tensor("op_346"), val = tensor(0x1p+0)]; + tensor var_347 = add(x = clip_5, y = var_346)[name = tensor("op_347")]; + tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_347)[name = tensor("clip_7")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_30, interleave = input_21_interleave_0, values = (window_3, window_5, window_7, window_9))[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; - tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; + tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; tensor inputs_3_pad_type_0 = const()[name = tensor("inputs_3_pad_type_0"), val = tensor("valid")]; tensor inputs_3_strides_0 = const()[name = tensor("inputs_3_strides_0"), val = tensor([1])]; tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_74")]; - tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_278_split_sizes_0 = const()[name = tensor("op_278_split_sizes_0"), val = tensor([256, 256])]; - tensor var_278_axis_0 = const()[name = tensor("op_278_axis_0"), val = tensor(1)]; - tensor var_278_0, tensor var_278_1 = split(axis = var_278_axis_0, split_sizes = var_278_split_sizes_0, x = inputs_3)[name = tensor("op_278")]; - tensor var_280 = sigmoid(x = var_278_1)[name = tensor("op_280")]; - tensor inputs_5 = mul(x = var_278_0, y = var_280)[name = tensor("inputs_5")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; + tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; + tensor var_370_split_sizes_0 = const()[name = tensor("op_370_split_sizes_0"), val = tensor([256, 256])]; + tensor var_370_axis_0 = const()[name = tensor("op_370_axis_0"), val = tensor(1)]; + tensor var_370_0, tensor var_370_1 = split(axis = var_370_axis_0, split_sizes = var_370_split_sizes_0, x = inputs_3)[name = tensor("op_370")]; + tensor var_372 = sigmoid(x = var_370_1)[name = tensor("op_372")]; + tensor inputs_5 = mul(x = var_370_0, y = var_372)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; tensor outputs_aug_1_strides_0 = const()[name = tensor("outputs_aug_1_strides_0"), val = tensor([1])]; tensor outputs_aug_1_dilations_0 = const()[name = tensor("outputs_aug_1_dilations_0"), val = tensor([1])]; - tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; + tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; tensor input_25_begin_0 = const()[name = tensor("input_25_begin_0"), val = tensor([0, 0, 0])]; - tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([1, 256, 16])]; + tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([4, 256, 16])]; tensor input_25_end_mask_0 = const()[name = tensor("input_25_end_mask_0"), val = tensor([true, true, false])]; - tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; - tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; - tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; + tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; + tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; + tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; tensor input_29_pad_type_0 = const()[name = tensor("input_29_pad_type_0"), val = tensor("valid")]; tensor input_29_strides_0 = const()[name = tensor("input_29_strides_0"), val = tensor([1])]; tensor input_29_pad_0 = const()[name = tensor("input_29_pad_0"), val = tensor([0, 0])]; tensor input_29_dilations_0 = const()[name = tensor("input_29_dilations_0"), val = tensor([1])]; tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; - tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; + tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_311_begin_0 = const()[name = tensor("op_311_begin_0"), val = tensor([0, -1, 0])]; - tensor var_311_end_0 = const()[name = tensor("op_311_end_0"), val = tensor([1, 16, 256])]; - tensor var_311_end_mask_0 = const()[name = tensor("op_311_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_73")]; - tensor var_311 = slice_by_index(begin = var_311_begin_0, end = var_311_end_0, end_mask = var_311_end_mask_0, x = conv_out_1)[name = tensor("op_311")]; - tensor var_314_begin_0 = const()[name = tensor("op_314_begin_0"), val = tensor([0, 1, 0])]; - tensor var_314_end_0 = const()[name = tensor("op_314_end_0"), val = tensor([1, 2, 256])]; - tensor var_314_end_mask_0 = const()[name = tensor("op_314_end_mask_0"), val = tensor([true, false, true])]; - tensor var_314 = slice_by_index(begin = var_314_begin_0, end = var_314_end_0, end_mask = var_314_end_mask_0, x = x_3)[name = tensor("op_314")]; - tensor var_317_begin_0 = const()[name = tensor("op_317_begin_0"), val = tensor([0, 1, 0])]; - tensor var_317_end_0 = const()[name = tensor("op_317_end_0"), val = tensor([1, 16, 256])]; - tensor var_317_end_mask_0 = const()[name = tensor("op_317_end_mask_0"), val = tensor([true, true, true])]; - tensor var_317 = slice_by_index(begin = var_317_begin_0, end = var_317_end_0, end_mask = var_317_end_mask_0, x = input_21)[name = tensor("op_317")]; - tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_317, var_314))[name = tensor("rolled_3")]; - tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; - tensor var_322 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_322")]; - tensor var_323_dtype_0 = const()[name = tensor("op_323_dtype_0"), val = tensor("fp32")]; - tensor var_324 = const()[name = tensor("op_324"), val = tensor([1, 16, 1])]; - tensor var_323 = cast(dtype = var_323_dtype_0, x = var_322)[name = tensor("cast_173")]; - tensor fill_mask_3 = reshape(shape = var_324, x = var_323)[name = tensor("fill_mask_3")]; - tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_314)[name = tensor("x_expanded_3")]; - tensor var_328 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_328")]; - tensor var_329 = mul(x = input_21, y = var_328)[name = tensor("op_329")]; - tensor var_330 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_330")]; - tensor filled_window_3 = add(x = var_329, y = var_330)[name = tensor("filled_window_3")]; - tensor var_332 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_332")]; - tensor var_333_dtype_0 = const()[name = tensor("op_333_dtype_0"), val = tensor("fp32")]; - tensor var_334 = const()[name = tensor("op_334"), val = tensor([1, 1, 1])]; - tensor var_333 = cast(dtype = var_333_dtype_0, x = var_332)[name = tensor("cast_172")]; - tensor use_roll_3 = reshape(shape = var_334, x = var_333)[name = tensor("use_roll_3")]; - tensor var_336 = sub(x = var_17, y = use_roll_3)[name = tensor("op_336")]; - tensor var_337 = mul(x = filled_window_3, y = var_336)[name = tensor("op_337")]; - tensor var_338 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_338")]; - tensor input_31 = add(x = var_337, y = var_338)[name = tensor("input_31")]; - tensor var_340 = const()[name = tensor("op_340"), val = tensor(0x1p+0)]; - tensor var_341 = add(x = clip_1, y = var_340)[name = tensor("op_341")]; - tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_341)[name = tensor("clip_3")]; + tensor var_403_begin_0 = const()[name = tensor("op_403_begin_0"), val = tensor([0, -1, 0])]; + tensor var_403_end_0 = const()[name = tensor("op_403_end_0"), val = tensor([4, 16, 256])]; + tensor var_403_end_mask_0 = const()[name = tensor("op_403_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_403 = slice_by_index(begin = var_403_begin_0, end = var_403_end_0, end_mask = var_403_end_mask_0, x = conv_out_1)[name = tensor("op_403")]; + tensor var_405_perm_0 = const()[name = tensor("op_405_perm_0"), val = tensor([1, 0, 2])]; + tensor var_405 = transpose(perm = var_405_perm_0, x = var_403)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_405)[name = tensor("input_31")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; + tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; + tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; + tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; + tensor var_428 = const()[name = tensor("op_428"), val = tensor(0x1p-1)]; + tensor var_429 = mul(x = input_39, y = var_428)[name = tensor("op_429")]; + tensor input_41 = add(x = var_429, y = input_31)[name = tensor("input_41")]; + tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; + tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; + tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; + tensor input_45 = layer_norm(axes = input_45_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_43)[name = tensor("input_45")]; + tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; + tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; + tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; + tensor var_458 = const()[name = tensor("op_458"), val = tensor(0x1p-1)]; + tensor var_459 = mul(x = input_51, y = var_458)[name = tensor("op_459")]; + tensor input_53 = add(x = var_459, y = input_43)[name = tensor("input_53")]; tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; - tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_31)[name = tensor("x_7")]; - tensor input_33_perm_0 = const()[name = tensor("input_33_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_9_pad_type_0 = const()[name = tensor("inputs_9_pad_type_0"), val = tensor("valid")]; - tensor inputs_9_strides_0 = const()[name = tensor("inputs_9_strides_0"), val = tensor([1])]; - tensor inputs_9_pad_0 = const()[name = tensor("inputs_9_pad_0"), val = tensor([0, 0])]; - tensor inputs_9_dilations_0 = const()[name = tensor("inputs_9_dilations_0"), val = tensor([1])]; - tensor inputs_9_groups_0 = const()[name = tensor("inputs_9_groups_0"), val = tensor(1)]; - tensor input_33 = transpose(perm = input_33_perm_0, x = x_7)[name = tensor("transpose_72")]; - tensor inputs_9 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_9_dilations_0, groups = inputs_9_groups_0, pad = inputs_9_pad_0, pad_type = inputs_9_pad_type_0, strides = inputs_9_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_33)[name = tensor("inputs_9")]; - tensor var_362_split_sizes_0 = const()[name = tensor("op_362_split_sizes_0"), val = tensor([256, 256])]; - tensor var_362_axis_0 = const()[name = tensor("op_362_axis_0"), val = tensor(1)]; - tensor var_362_0, tensor var_362_1 = split(axis = var_362_axis_0, split_sizes = var_362_split_sizes_0, x = inputs_9)[name = tensor("op_362")]; - tensor var_364 = sigmoid(x = var_362_1)[name = tensor("op_364")]; - tensor inputs_11 = mul(x = var_362_0, y = var_364)[name = tensor("inputs_11")]; - tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; - tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; - tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; - tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_11)[name = tensor("outputs_aug_3")]; - tensor input_35_begin_0 = const()[name = tensor("input_35_begin_0"), val = tensor([0, 0, 0])]; - tensor input_35_end_0 = const()[name = tensor("input_35_end_0"), val = tensor([1, 256, 16])]; - tensor input_35_end_mask_0 = const()[name = tensor("input_35_end_mask_0"), val = tensor([true, true, false])]; - tensor input_35 = slice_by_index(begin = input_35_begin_0, end = input_35_end_0, end_mask = input_35_end_mask_0, x = outputs_aug_3)[name = tensor("input_35")]; - tensor inputs_13 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_35)[name = tensor("inputs_13")]; - tensor input_37 = silu(x = inputs_13)[name = tensor("input_37")]; - tensor input_39_pad_type_0 = const()[name = tensor("input_39_pad_type_0"), val = tensor("valid")]; - tensor input_39_strides_0 = const()[name = tensor("input_39_strides_0"), val = tensor([1])]; - tensor input_39_pad_0 = const()[name = tensor("input_39_pad_0"), val = tensor([0, 0])]; - tensor input_39_dilations_0 = const()[name = tensor("input_39_dilations_0"), val = tensor([1])]; - tensor input_39_groups_0 = const()[name = tensor("input_39_groups_0"), val = tensor(1)]; - tensor input_39 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_37)[name = tensor("input_39")]; - tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_395_begin_0 = const()[name = tensor("op_395_begin_0"), val = tensor([0, -1, 0])]; - tensor var_395_end_0 = const()[name = tensor("op_395_end_0"), val = tensor([1, 16, 256])]; - tensor var_395_end_mask_0 = const()[name = tensor("op_395_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_39)[name = tensor("transpose_71")]; - tensor var_395 = slice_by_index(begin = var_395_begin_0, end = var_395_end_0, end_mask = var_395_end_mask_0, x = conv_out_3)[name = tensor("op_395")]; - tensor var_398_begin_0 = const()[name = tensor("op_398_begin_0"), val = tensor([0, 2, 0])]; - tensor var_398_end_0 = const()[name = tensor("op_398_end_0"), val = tensor([1, 3, 256])]; - tensor var_398_end_mask_0 = const()[name = tensor("op_398_end_mask_0"), val = tensor([true, false, true])]; - tensor var_398 = slice_by_index(begin = var_398_begin_0, end = var_398_end_0, end_mask = var_398_end_mask_0, x = x_3)[name = tensor("op_398")]; - tensor var_401_begin_0 = const()[name = tensor("op_401_begin_0"), val = tensor([0, 1, 0])]; - tensor var_401_end_0 = const()[name = tensor("op_401_end_0"), val = tensor([1, 16, 256])]; - tensor var_401_end_mask_0 = const()[name = tensor("op_401_end_mask_0"), val = tensor([true, true, true])]; - tensor var_401 = slice_by_index(begin = var_401_begin_0, end = var_401_end_0, end_mask = var_401_end_mask_0, x = input_31)[name = tensor("op_401")]; - tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_401, var_398))[name = tensor("rolled_5")]; - tensor clip_4 = clip(alpha = var_21, beta = var_20, x = clip_3)[name = tensor("clip_4")]; - tensor var_406 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_406")]; - tensor var_407_dtype_0 = const()[name = tensor("op_407_dtype_0"), val = tensor("fp32")]; - tensor var_408 = const()[name = tensor("op_408"), val = tensor([1, 16, 1])]; - tensor var_407 = cast(dtype = var_407_dtype_0, x = var_406)[name = tensor("cast_171")]; - tensor fill_mask_5 = reshape(shape = var_408, x = var_407)[name = tensor("fill_mask_5")]; - tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_398)[name = tensor("x_expanded_5")]; - tensor var_412 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_412")]; - tensor var_413 = mul(x = input_31, y = var_412)[name = tensor("op_413")]; - tensor var_414 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_414")]; - tensor filled_window_5 = add(x = var_413, y = var_414)[name = tensor("filled_window_5")]; - tensor var_416 = greater_equal(x = clip_3, y = var_16)[name = tensor("op_416")]; - tensor var_417_dtype_0 = const()[name = tensor("op_417_dtype_0"), val = tensor("fp32")]; - tensor var_418 = const()[name = tensor("op_418"), val = tensor([1, 1, 1])]; - tensor var_417 = cast(dtype = var_417_dtype_0, x = var_416)[name = tensor("cast_170")]; - tensor use_roll_5 = reshape(shape = var_418, x = var_417)[name = tensor("use_roll_5")]; - tensor var_420 = sub(x = var_17, y = use_roll_5)[name = tensor("op_420")]; - tensor var_421 = mul(x = filled_window_5, y = var_420)[name = tensor("op_421")]; - tensor var_422 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_422")]; - tensor input_41 = add(x = var_421, y = var_422)[name = tensor("input_41")]; - tensor var_424 = const()[name = tensor("op_424"), val = tensor(0x1p+0)]; - tensor var_425 = add(x = clip_3, y = var_424)[name = tensor("op_425")]; - tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_425)[name = tensor("clip_5")]; - tensor x_9_axes_0 = const()[name = tensor("x_9_axes_0"), val = tensor([-1])]; - tensor x_9 = layer_norm(axes = x_9_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_41)[name = tensor("x_9")]; - tensor input_43_perm_0 = const()[name = tensor("input_43_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_15_pad_type_0 = const()[name = tensor("inputs_15_pad_type_0"), val = tensor("valid")]; - tensor inputs_15_strides_0 = const()[name = tensor("inputs_15_strides_0"), val = tensor([1])]; - tensor inputs_15_pad_0 = const()[name = tensor("inputs_15_pad_0"), val = tensor([0, 0])]; - tensor inputs_15_dilations_0 = const()[name = tensor("inputs_15_dilations_0"), val = tensor([1])]; - tensor inputs_15_groups_0 = const()[name = tensor("inputs_15_groups_0"), val = tensor(1)]; - tensor input_43 = transpose(perm = input_43_perm_0, x = x_9)[name = tensor("transpose_70")]; - tensor inputs_15 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_15_dilations_0, groups = inputs_15_groups_0, pad = inputs_15_pad_0, pad_type = inputs_15_pad_type_0, strides = inputs_15_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_43)[name = tensor("inputs_15")]; - tensor var_446_split_sizes_0 = const()[name = tensor("op_446_split_sizes_0"), val = tensor([256, 256])]; - tensor var_446_axis_0 = const()[name = tensor("op_446_axis_0"), val = tensor(1)]; - tensor var_446_0, tensor var_446_1 = split(axis = var_446_axis_0, split_sizes = var_446_split_sizes_0, x = inputs_15)[name = tensor("op_446")]; - tensor var_448 = sigmoid(x = var_446_1)[name = tensor("op_448")]; - tensor inputs_17 = mul(x = var_446_0, y = var_448)[name = tensor("inputs_17")]; - tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; - tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; - tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; - tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_17)[name = tensor("outputs_aug_5")]; - tensor input_45_begin_0 = const()[name = tensor("input_45_begin_0"), val = tensor([0, 0, 0])]; - tensor input_45_end_0 = const()[name = tensor("input_45_end_0"), val = tensor([1, 256, 16])]; - tensor input_45_end_mask_0 = const()[name = tensor("input_45_end_mask_0"), val = tensor([true, true, false])]; - tensor input_45 = slice_by_index(begin = input_45_begin_0, end = input_45_end_0, end_mask = input_45_end_mask_0, x = outputs_aug_5)[name = tensor("input_45")]; - tensor inputs_19 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_45)[name = tensor("inputs_19")]; - tensor input_47 = silu(x = inputs_19)[name = tensor("input_47")]; - tensor input_49_pad_type_0 = const()[name = tensor("input_49_pad_type_0"), val = tensor("valid")]; - tensor input_49_strides_0 = const()[name = tensor("input_49_strides_0"), val = tensor([1])]; - tensor input_49_pad_0 = const()[name = tensor("input_49_pad_0"), val = tensor([0, 0])]; - tensor input_49_dilations_0 = const()[name = tensor("input_49_dilations_0"), val = tensor([1])]; - tensor input_49_groups_0 = const()[name = tensor("input_49_groups_0"), val = tensor(1)]; - tensor input_49 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_49_dilations_0, groups = input_49_groups_0, pad = input_49_pad_0, pad_type = input_49_pad_type_0, strides = input_49_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_47)[name = tensor("input_49")]; - tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_479_begin_0 = const()[name = tensor("op_479_begin_0"), val = tensor([0, -1, 0])]; - tensor var_479_end_0 = const()[name = tensor("op_479_end_0"), val = tensor([1, 16, 256])]; - tensor var_479_end_mask_0 = const()[name = tensor("op_479_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_49)[name = tensor("transpose_69")]; - tensor var_479 = slice_by_index(begin = var_479_begin_0, end = var_479_end_0, end_mask = var_479_end_mask_0, x = conv_out_5)[name = tensor("op_479")]; - tensor var_482_begin_0 = const()[name = tensor("op_482_begin_0"), val = tensor([0, 3, 0])]; - tensor var_482_end_0 = const()[name = tensor("op_482_end_0"), val = tensor([1, 1, 256])]; - tensor var_482_end_mask_0 = const()[name = tensor("op_482_end_mask_0"), val = tensor([true, true, true])]; - tensor var_482 = slice_by_index(begin = var_482_begin_0, end = var_482_end_0, end_mask = var_482_end_mask_0, x = x_3)[name = tensor("op_482")]; - tensor var_485_begin_0 = const()[name = tensor("op_485_begin_0"), val = tensor([0, 1, 0])]; - tensor var_485_end_0 = const()[name = tensor("op_485_end_0"), val = tensor([1, 16, 256])]; - tensor var_485_end_mask_0 = const()[name = tensor("op_485_end_mask_0"), val = tensor([true, true, true])]; - tensor var_485 = slice_by_index(begin = var_485_begin_0, end = var_485_end_0, end_mask = var_485_end_mask_0, x = input_41)[name = tensor("op_485")]; - tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; - tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_485, var_482))[name = tensor("rolled_7")]; - tensor clip_6 = clip(alpha = var_21, beta = var_20, x = clip_5)[name = tensor("clip_6")]; - tensor var_490 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_490")]; - tensor var_491_dtype_0 = const()[name = tensor("op_491_dtype_0"), val = tensor("fp32")]; - tensor var_492 = const()[name = tensor("op_492"), val = tensor([1, 16, 1])]; - tensor var_491 = cast(dtype = var_491_dtype_0, x = var_490)[name = tensor("cast_169")]; - tensor fill_mask_7 = reshape(shape = var_492, x = var_491)[name = tensor("fill_mask_7")]; - tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_482)[name = tensor("x_expanded_7")]; - tensor var_496 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_496")]; - tensor var_497 = mul(x = input_41, y = var_496)[name = tensor("op_497")]; - tensor var_498 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_498")]; - tensor filled_window_7 = add(x = var_497, y = var_498)[name = tensor("filled_window_7")]; - tensor var_500 = greater_equal(x = clip_5, y = var_16)[name = tensor("op_500")]; - tensor var_501_dtype_0 = const()[name = tensor("op_501_dtype_0"), val = tensor("fp32")]; - tensor var_502 = const()[name = tensor("op_502"), val = tensor([1, 1, 1])]; - tensor var_501 = cast(dtype = var_501_dtype_0, x = var_500)[name = tensor("cast_168")]; - tensor use_roll_7 = reshape(shape = var_502, x = var_501)[name = tensor("use_roll_7")]; - tensor var_504 = sub(x = var_17, y = use_roll_7)[name = tensor("op_504")]; - tensor var_505 = mul(x = filled_window_7, y = var_504)[name = tensor("op_505")]; - tensor var_506 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_506")]; - tensor input_51 = add(x = var_505, y = var_506)[name = tensor("input_51")]; - tensor var_508 = const()[name = tensor("op_508"), val = tensor(0x1p+0)]; - tensor var_509 = add(x = clip_5, y = var_508)[name = tensor("op_509")]; - tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_509)[name = tensor("clip_7")]; - tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; - tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_51)[name = tensor("x_11")]; - tensor input_53_perm_0 = const()[name = tensor("input_53_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_21_pad_type_0 = const()[name = tensor("inputs_21_pad_type_0"), val = tensor("valid")]; - tensor inputs_21_strides_0 = const()[name = tensor("inputs_21_strides_0"), val = tensor([1])]; - tensor inputs_21_pad_0 = const()[name = tensor("inputs_21_pad_0"), val = tensor([0, 0])]; - tensor inputs_21_dilations_0 = const()[name = tensor("inputs_21_dilations_0"), val = tensor([1])]; - tensor inputs_21_groups_0 = const()[name = tensor("inputs_21_groups_0"), val = tensor(1)]; - tensor input_53 = transpose(perm = input_53_perm_0, x = x_11)[name = tensor("transpose_68")]; - tensor inputs_21 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_21_dilations_0, groups = inputs_21_groups_0, pad = inputs_21_pad_0, pad_type = inputs_21_pad_type_0, strides = inputs_21_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_53)[name = tensor("inputs_21")]; - tensor var_530_split_sizes_0 = const()[name = tensor("op_530_split_sizes_0"), val = tensor([256, 256])]; - tensor var_530_axis_0 = const()[name = tensor("op_530_axis_0"), val = tensor(1)]; - tensor var_530_0, tensor var_530_1 = split(axis = var_530_axis_0, split_sizes = var_530_split_sizes_0, x = inputs_21)[name = tensor("op_530")]; - tensor var_532 = sigmoid(x = var_530_1)[name = tensor("op_532")]; - tensor inputs_23 = mul(x = var_530_0, y = var_532)[name = tensor("inputs_23")]; - tensor outputs_aug_7_pad_type_0 = const()[name = tensor("outputs_aug_7_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_7_pad_0 = const()[name = tensor("outputs_aug_7_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_7_groups_0 = const()[name = tensor("outputs_aug_7_groups_0"), val = tensor(256)]; - tensor outputs_aug_7_strides_0 = const()[name = tensor("outputs_aug_7_strides_0"), val = tensor([1])]; - tensor outputs_aug_7_dilations_0 = const()[name = tensor("outputs_aug_7_dilations_0"), val = tensor([1])]; - tensor outputs_aug_7 = conv(dilations = outputs_aug_7_dilations_0, groups = outputs_aug_7_groups_0, pad = outputs_aug_7_pad_0, pad_type = outputs_aug_7_pad_type_0, strides = outputs_aug_7_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_23)[name = tensor("outputs_aug_7")]; - tensor input_55_begin_0 = const()[name = tensor("input_55_begin_0"), val = tensor([0, 0, 0])]; - tensor input_55_end_0 = const()[name = tensor("input_55_end_0"), val = tensor([1, 256, 16])]; - tensor input_55_end_mask_0 = const()[name = tensor("input_55_end_mask_0"), val = tensor([true, true, false])]; - tensor input_55 = slice_by_index(begin = input_55_begin_0, end = input_55_end_0, end_mask = input_55_end_mask_0, x = outputs_aug_7)[name = tensor("input_55")]; - tensor inputs_25 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_55)[name = tensor("inputs_25")]; - tensor input_57 = silu(x = inputs_25)[name = tensor("input_57")]; - tensor input_59_pad_type_0 = const()[name = tensor("input_59_pad_type_0"), val = tensor("valid")]; - tensor input_59_strides_0 = const()[name = tensor("input_59_strides_0"), val = tensor([1])]; - tensor input_59_pad_0 = const()[name = tensor("input_59_pad_0"), val = tensor([0, 0])]; - tensor input_59_dilations_0 = const()[name = tensor("input_59_dilations_0"), val = tensor([1])]; - tensor input_59_groups_0 = const()[name = tensor("input_59_groups_0"), val = tensor(1)]; - tensor input_59 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_59_dilations_0, groups = input_59_groups_0, pad = input_59_pad_0, pad_type = input_59_pad_type_0, strides = input_59_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_57)[name = tensor("input_59")]; - tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_563_begin_0 = const()[name = tensor("op_563_begin_0"), val = tensor([0, -1, 0])]; - tensor var_563_end_0 = const()[name = tensor("op_563_end_0"), val = tensor([1, 16, 256])]; - tensor var_563_end_mask_0 = const()[name = tensor("op_563_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_59)[name = tensor("transpose_67")]; - tensor var_563 = slice_by_index(begin = var_563_begin_0, end = var_563_end_0, end_mask = var_563_end_mask_0, x = conv_out_7)[name = tensor("op_563")]; - tensor var_566_interleave_0 = const()[name = tensor("op_566_interleave_0"), val = tensor(false)]; - tensor var_566 = concat(axis = var_33, interleave = var_566_interleave_0, values = (var_311, var_395, var_479, var_563))[name = tensor("op_566")]; - tensor input_61 = add(x = x_3, y = var_566)[name = tensor("input_61")]; - tensor input_63_axes_0 = const()[name = tensor("input_63_axes_0"), val = tensor([-1])]; - tensor input_63 = layer_norm(axes = input_63_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_61)[name = tensor("input_63")]; - tensor inputs_27 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_63)[name = tensor("linear_8")]; - tensor input_65 = silu(x = inputs_27)[name = tensor("input_65")]; - tensor input_69 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_65)[name = tensor("linear_9")]; - tensor var_589 = const()[name = tensor("op_589"), val = tensor(0x1p-1)]; - tensor var_590 = mul(x = input_69, y = var_589)[name = tensor("op_590")]; - tensor input_71 = add(x = var_590, y = input_61)[name = tensor("input_71")]; - tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; - tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_71)[name = tensor("input_73")]; - tensor input_75_axes_0 = const()[name = tensor("input_75_axes_0"), val = tensor([-1])]; - tensor input_75 = layer_norm(axes = input_75_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_73)[name = tensor("input_75")]; - tensor inputs_29 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_75)[name = tensor("linear_10")]; - tensor input_77 = silu(x = inputs_29)[name = tensor("input_77")]; - tensor input_81 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_77)[name = tensor("linear_11")]; - tensor var_619 = const()[name = tensor("op_619"), val = tensor(0x1p-1)]; - tensor var_620 = mul(x = input_81, y = var_619)[name = tensor("op_620")]; - tensor input_83 = add(x = var_620, y = input_73)[name = tensor("input_83")]; - tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; - tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_83)[name = tensor("x_13")]; + tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_3_end_0 = const()[name = tensor("prev_kv_3_end_0"), val = tensor([2, 1, 4, 64, 64])]; tensor prev_kv_3_end_mask_0 = const()[name = tensor("prev_kv_3_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -636,389 +522,274 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_634 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_13)[name = tensor("linear_12")]; - tensor var_635 = const()[name = tensor("op_635"), val = tensor([1, 4, 4, 64])]; - tensor var_636 = reshape(shape = var_635, x = var_634)[name = tensor("op_636")]; + tensor var_473 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_474 = const()[name = tensor("op_474"), val = tensor([1, 4, 4, 64])]; + tensor var_475 = reshape(shape = var_474, x = var_473)[name = tensor("op_475")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_640 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_13)[name = tensor("linear_13")]; - tensor var_641 = const()[name = tensor("op_641"), val = tensor(0x1p-3)]; - tensor var_642 = mul(x = var_640, y = var_641)[name = tensor("op_642")]; - tensor var_643 = const()[name = tensor("op_643"), val = tensor([1, 4, 4, 64])]; - tensor var_644 = reshape(shape = var_643, x = var_642)[name = tensor("op_644")]; + tensor var_479 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_480 = const()[name = tensor("op_480"), val = tensor(0x1p-3)]; + tensor var_481 = mul(x = var_479, y = var_480)[name = tensor("op_481")]; + tensor var_482 = const()[name = tensor("op_482"), val = tensor([1, 4, 4, 64])]; + tensor var_483 = reshape(shape = var_482, x = var_481)[name = tensor("op_483")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_648 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_13)[name = tensor("linear_14")]; - tensor var_649 = const()[name = tensor("op_649"), val = tensor([1, 4, 4, 64])]; - tensor var_650 = reshape(shape = var_649, x = var_648)[name = tensor("op_650")]; + tensor var_487 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_488 = const()[name = tensor("op_488"), val = tensor([1, 4, 4, 64])]; + tensor var_489 = reshape(shape = var_488, x = var_487)[name = tensor("op_489")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_87 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_13)[name = tensor("linear_15")]; + tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; tensor s_t_3 = add(x = prev_scale_3, y = encoder__t_index)[name = tensor("s_t_3")]; tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_644)[name = tensor("transpose_65")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_636)[name = tensor("transpose_66")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_483)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_475)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_660 = const()[name = tensor("op_660"), val = tensor([4, 1])]; - tensor var_661 = reshape(shape = var_660, x = sqrt_s_t_3)[name = tensor("op_661")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_661)[name = tensor("M_3")]; - tensor var_663 = mul(x = qk_3, y = M_3)[name = tensor("op_663")]; + tensor var_499 = const()[name = tensor("op_499"), val = tensor([4, 1])]; + tensor var_500 = reshape(shape = var_499, x = sqrt_s_t_3)[name = tensor("op_500")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_500)[name = tensor("M_3")]; + tensor var_502 = mul(x = qk_3, y = M_3)[name = tensor("op_502")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_650)[name = tensor("transpose_64")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_663, y = v_3)[name = tensor("inner_3")]; - tensor var_665_transpose_x_0 = const()[name = tensor("op_665_transpose_x_0"), val = tensor(false)]; - tensor var_665_transpose_y_0 = const()[name = tensor("op_665_transpose_y_0"), val = tensor(false)]; - tensor var_665 = matmul(transpose_x = var_665_transpose_x_0, transpose_y = var_665_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_665")]; - tensor var_666 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_666")]; - tensor var_667 = const()[name = tensor("op_667"), val = tensor([1, 1, 4, 1])]; - tensor var_668 = reshape(shape = var_667, x = var_666)[name = tensor("op_668")]; - tensor cross_3 = mul(x = var_665, y = var_668)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_489)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_502, y = v_3)[name = tensor("inner_3")]; + tensor var_504_transpose_x_0 = const()[name = tensor("op_504_transpose_x_0"), val = tensor(false)]; + tensor var_504_transpose_y_0 = const()[name = tensor("op_504_transpose_y_0"), val = tensor(false)]; + tensor var_504 = matmul(transpose_x = var_504_transpose_x_0, transpose_y = var_504_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_504")]; + tensor var_505 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_505")]; + tensor var_506 = const()[name = tensor("op_506"), val = tensor([1, 1, 4, 1])]; + tensor var_507 = reshape(shape = var_506, x = var_505)[name = tensor("op_507")]; + tensor cross_3 = mul(x = var_504, y = var_507)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_671 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_671")]; - tensor var_673_transpose_x_1 = const()[name = tensor("op_673_transpose_x_1"), val = tensor(true)]; - tensor var_673_transpose_y_1 = const()[name = tensor("op_673_transpose_y_1"), val = tensor(false)]; - tensor var_673 = matmul(transpose_x = var_673_transpose_x_1, transpose_y = var_673_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_673")]; - tensor new_kv_unnorm_3 = add(x = var_671, y = var_673)[name = tensor("new_kv_unnorm_3")]; - tensor var_675 = const()[name = tensor("op_675"), val = tensor(0x1p+2)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_675)[name = tensor("new_scale_3")]; - tensor var_677 = sqrt(x = new_scale_3)[name = tensor("op_677")]; - tensor var_678 = real_div(x = new_kv_unnorm_3, y = var_677)[name = tensor("op_678")]; - tensor var_679_perm_0 = const()[name = tensor("op_679_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_510 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_510")]; + tensor var_512_transpose_x_1 = const()[name = tensor("op_512_transpose_x_1"), val = tensor(true)]; + tensor var_512_transpose_y_1 = const()[name = tensor("op_512_transpose_y_1"), val = tensor(false)]; + tensor var_512 = matmul(transpose_x = var_512_transpose_x_1, transpose_y = var_512_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_512")]; + tensor new_kv_unnorm_3 = add(x = var_510, y = var_512)[name = tensor("new_kv_unnorm_3")]; + tensor var_514 = const()[name = tensor("op_514"), val = tensor(0x1p+2)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_514)[name = tensor("new_scale_3")]; + tensor var_516 = sqrt(x = new_scale_3)[name = tensor("op_516")]; + tensor var_517 = real_div(x = new_kv_unnorm_3, y = var_516)[name = tensor("op_517")]; + tensor var_518_perm_0 = const()[name = tensor("op_518_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_679 = transpose(perm = var_679_perm_0, x = out_7)[name = tensor("transpose_63")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_679)[name = tensor("out_9")]; - tensor var_683 = const()[name = tensor("op_683"), val = tensor([1, 4, 256])]; - tensor out_11 = reshape(shape = var_683, x = out_9)[name = tensor("out_11")]; - tensor var_685 = silu(x = input_87)[name = tensor("op_685")]; - tensor input_89 = mul(x = var_685, y = out_11)[name = tensor("input_89")]; - tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_89)[name = tensor("linear_16")]; - tensor x_15 = add(x = input_83, y = ret_out_3)[name = tensor("x_15")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor var_518 = transpose(perm = var_518_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_518)[name = tensor("out_9")]; + tensor var_522 = const()[name = tensor("op_522"), val = tensor([1, 4, 256])]; + tensor out_11 = reshape(shape = var_522, x = out_9)[name = tensor("out_11")]; + tensor var_524 = silu(x = input_57)[name = tensor("op_524")]; + tensor input_59 = mul(x = var_524, y = out_11)[name = tensor("input_59")]; + tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; + tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; + tensor window_11_begin_0 = const()[name = tensor("window_11_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_11_end_0 = const()[name = tensor("window_11_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_11_end_mask_0 = const()[name = tensor("window_11_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_11_squeeze_mask_0 = const()[name = tensor("window_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_11 = slice_by_index(begin = window_11_begin_0, end = window_11_end_0, end_mask = window_11_end_mask_0, squeeze_mask = window_11_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_11")]; tensor filled_9_begin_0 = const()[name = tensor("filled_9_begin_0"), val = tensor([1, 0])]; tensor filled_9_end_0 = const()[name = tensor("filled_9_end_0"), val = tensor([2, 1])]; tensor filled_9_end_mask_0 = const()[name = tensor("filled_9_end_mask_0"), val = tensor([false, true])]; tensor filled_9_squeeze_mask_0 = const()[name = tensor("filled_9_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_9 = slice_by_index(begin = filled_9_begin_0, end = filled_9_end_0, end_mask = filled_9_end_mask_0, squeeze_mask = filled_9_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_9")]; - tensor var_694_begin_0 = const()[name = tensor("op_694_begin_0"), val = tensor([0, 0, 0])]; - tensor var_694_end_0 = const()[name = tensor("op_694_end_0"), val = tensor([1, 1, 256])]; - tensor var_694_end_mask_0 = const()[name = tensor("op_694_end_mask_0"), val = tensor([true, false, true])]; - tensor var_694 = slice_by_index(begin = var_694_begin_0, end = var_694_end_0, end_mask = var_694_end_mask_0, x = x_15)[name = tensor("op_694")]; - tensor var_697_begin_0 = const()[name = tensor("op_697_begin_0"), val = tensor([0, 1, 0])]; - tensor var_697_end_0 = const()[name = tensor("op_697_end_0"), val = tensor([1, 16, 256])]; - tensor var_697_end_mask_0 = const()[name = tensor("op_697_end_mask_0"), val = tensor([true, true, true])]; - tensor var_697 = slice_by_index(begin = var_697_begin_0, end = var_697_end_0, end_mask = var_697_end_mask_0, x = window_3)[name = tensor("op_697")]; + tensor var_533_begin_0 = const()[name = tensor("op_533_begin_0"), val = tensor([0, 0, 0])]; + tensor var_533_end_0 = const()[name = tensor("op_533_end_0"), val = tensor([1, 1, 256])]; + tensor var_533_end_mask_0 = const()[name = tensor("op_533_end_mask_0"), val = tensor([true, false, true])]; + tensor var_533 = slice_by_index(begin = var_533_begin_0, end = var_533_end_0, end_mask = var_533_end_mask_0, x = x_9)[name = tensor("op_533")]; + tensor var_536_begin_0 = const()[name = tensor("op_536_begin_0"), val = tensor([0, 1, 0])]; + tensor var_536_end_0 = const()[name = tensor("op_536_end_0"), val = tensor([1, 16, 256])]; + tensor var_536_end_mask_0 = const()[name = tensor("op_536_end_mask_0"), val = tensor([true, true, true])]; + tensor var_536 = slice_by_index(begin = var_536_begin_0, end = var_536_end_0, end_mask = var_536_end_mask_0, x = window_11)[name = tensor("op_536")]; tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; - tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_697, var_694))[name = tensor("rolled_9")]; + tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_536, var_533))[name = tensor("rolled_9")]; tensor clip_8 = clip(alpha = var_21, beta = var_20, x = filled_9)[name = tensor("clip_8")]; - tensor var_702 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_702")]; - tensor var_703_dtype_0 = const()[name = tensor("op_703_dtype_0"), val = tensor("fp32")]; - tensor var_704 = const()[name = tensor("op_704"), val = tensor([1, 16, 1])]; - tensor var_703 = cast(dtype = var_703_dtype_0, x = var_702)[name = tensor("cast_167")]; - tensor fill_mask_9 = reshape(shape = var_704, x = var_703)[name = tensor("fill_mask_9")]; + tensor var_541 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_541")]; + tensor var_542_dtype_0 = const()[name = tensor("op_542_dtype_0"), val = tensor("fp32")]; + tensor var_543 = const()[name = tensor("op_543"), val = tensor([1, 16, 1])]; + tensor var_542 = cast(dtype = var_542_dtype_0, x = var_541)[name = tensor("cast_167")]; + tensor fill_mask_9 = reshape(shape = var_543, x = var_542)[name = tensor("fill_mask_9")]; tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_694)[name = tensor("x_expanded_9")]; - tensor var_708 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_708")]; - tensor var_709 = mul(x = window_3, y = var_708)[name = tensor("op_709")]; - tensor var_710 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_710")]; - tensor filled_window_9 = add(x = var_709, y = var_710)[name = tensor("filled_window_9")]; - tensor var_712 = greater_equal(x = filled_9, y = var_16)[name = tensor("op_712")]; - tensor var_713_dtype_0 = const()[name = tensor("op_713_dtype_0"), val = tensor("fp32")]; - tensor var_714 = const()[name = tensor("op_714"), val = tensor([1, 1, 1])]; - tensor var_713 = cast(dtype = var_713_dtype_0, x = var_712)[name = tensor("cast_166")]; - tensor use_roll_9 = reshape(shape = var_714, x = var_713)[name = tensor("use_roll_9")]; - tensor var_716 = sub(x = var_17, y = use_roll_9)[name = tensor("op_716")]; - tensor var_717 = mul(x = filled_window_9, y = var_716)[name = tensor("op_717")]; - tensor var_718 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_718")]; - tensor input_91 = add(x = var_717, y = var_718)[name = tensor("input_91")]; - tensor var_720 = const()[name = tensor("op_720"), val = tensor(0x1p+0)]; - tensor var_721 = add(x = filled_9, y = var_720)[name = tensor("op_721")]; - tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_721)[name = tensor("clip_9")]; - tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; - tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_91)[name = tensor("x_17")]; - tensor input_93_perm_0 = const()[name = tensor("input_93_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_31_pad_type_0 = const()[name = tensor("inputs_31_pad_type_0"), val = tensor("valid")]; - tensor inputs_31_strides_0 = const()[name = tensor("inputs_31_strides_0"), val = tensor([1])]; - tensor inputs_31_pad_0 = const()[name = tensor("inputs_31_pad_0"), val = tensor([0, 0])]; - tensor inputs_31_dilations_0 = const()[name = tensor("inputs_31_dilations_0"), val = tensor([1])]; - tensor inputs_31_groups_0 = const()[name = tensor("inputs_31_groups_0"), val = tensor(1)]; - tensor input_93 = transpose(perm = input_93_perm_0, x = x_17)[name = tensor("transpose_62")]; - tensor inputs_31 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_31_dilations_0, groups = inputs_31_groups_0, pad = inputs_31_pad_0, pad_type = inputs_31_pad_type_0, strides = inputs_31_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_93)[name = tensor("inputs_31")]; - tensor var_742_split_sizes_0 = const()[name = tensor("op_742_split_sizes_0"), val = tensor([256, 256])]; - tensor var_742_axis_0 = const()[name = tensor("op_742_axis_0"), val = tensor(1)]; - tensor var_742_0, tensor var_742_1 = split(axis = var_742_axis_0, split_sizes = var_742_split_sizes_0, x = inputs_31)[name = tensor("op_742")]; - tensor var_744 = sigmoid(x = var_742_1)[name = tensor("op_744")]; - tensor inputs_33 = mul(x = var_742_0, y = var_744)[name = tensor("inputs_33")]; - tensor outputs_aug_9_pad_type_0 = const()[name = tensor("outputs_aug_9_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_9_pad_0 = const()[name = tensor("outputs_aug_9_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_9_groups_0 = const()[name = tensor("outputs_aug_9_groups_0"), val = tensor(256)]; - tensor outputs_aug_9_strides_0 = const()[name = tensor("outputs_aug_9_strides_0"), val = tensor([1])]; - tensor outputs_aug_9_dilations_0 = const()[name = tensor("outputs_aug_9_dilations_0"), val = tensor([1])]; - tensor outputs_aug_9 = conv(dilations = outputs_aug_9_dilations_0, groups = outputs_aug_9_groups_0, pad = outputs_aug_9_pad_0, pad_type = outputs_aug_9_pad_type_0, strides = outputs_aug_9_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_33)[name = tensor("outputs_aug_9")]; - tensor input_95_begin_0 = const()[name = tensor("input_95_begin_0"), val = tensor([0, 0, 0])]; - tensor input_95_end_0 = const()[name = tensor("input_95_end_0"), val = tensor([1, 256, 16])]; - tensor input_95_end_mask_0 = const()[name = tensor("input_95_end_mask_0"), val = tensor([true, true, false])]; - tensor input_95 = slice_by_index(begin = input_95_begin_0, end = input_95_end_0, end_mask = input_95_end_mask_0, x = outputs_aug_9)[name = tensor("input_95")]; - tensor inputs_35 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_95)[name = tensor("inputs_35")]; - tensor input_97 = silu(x = inputs_35)[name = tensor("input_97")]; - tensor input_99_pad_type_0 = const()[name = tensor("input_99_pad_type_0"), val = tensor("valid")]; - tensor input_99_strides_0 = const()[name = tensor("input_99_strides_0"), val = tensor([1])]; - tensor input_99_pad_0 = const()[name = tensor("input_99_pad_0"), val = tensor([0, 0])]; - tensor input_99_dilations_0 = const()[name = tensor("input_99_dilations_0"), val = tensor([1])]; - tensor input_99_groups_0 = const()[name = tensor("input_99_groups_0"), val = tensor(1)]; - tensor input_99 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_99_dilations_0, groups = input_99_groups_0, pad = input_99_pad_0, pad_type = input_99_pad_type_0, strides = input_99_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_97)[name = tensor("input_99")]; - tensor conv_out_9_perm_0 = const()[name = tensor("conv_out_9_perm_0"), val = tensor([0, 2, 1])]; - tensor var_775_begin_0 = const()[name = tensor("op_775_begin_0"), val = tensor([0, -1, 0])]; - tensor var_775_end_0 = const()[name = tensor("op_775_end_0"), val = tensor([1, 16, 256])]; - tensor var_775_end_mask_0 = const()[name = tensor("op_775_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_9 = transpose(perm = conv_out_9_perm_0, x = input_99)[name = tensor("transpose_61")]; - tensor var_775 = slice_by_index(begin = var_775_begin_0, end = var_775_end_0, end_mask = var_775_end_mask_0, x = conv_out_9)[name = tensor("op_775")]; - tensor var_778_begin_0 = const()[name = tensor("op_778_begin_0"), val = tensor([0, 1, 0])]; - tensor var_778_end_0 = const()[name = tensor("op_778_end_0"), val = tensor([1, 2, 256])]; - tensor var_778_end_mask_0 = const()[name = tensor("op_778_end_mask_0"), val = tensor([true, false, true])]; - tensor var_778 = slice_by_index(begin = var_778_begin_0, end = var_778_end_0, end_mask = var_778_end_mask_0, x = x_15)[name = tensor("op_778")]; - tensor var_781_begin_0 = const()[name = tensor("op_781_begin_0"), val = tensor([0, 1, 0])]; - tensor var_781_end_0 = const()[name = tensor("op_781_end_0"), val = tensor([1, 16, 256])]; - tensor var_781_end_mask_0 = const()[name = tensor("op_781_end_mask_0"), val = tensor([true, true, true])]; - tensor var_781 = slice_by_index(begin = var_781_begin_0, end = var_781_end_0, end_mask = var_781_end_mask_0, x = input_91)[name = tensor("op_781")]; + tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_533)[name = tensor("x_expanded_9")]; + tensor var_547 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_547")]; + tensor var_548 = mul(x = window_11, y = var_547)[name = tensor("op_548")]; + tensor var_549 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_549")]; + tensor filled_window_9 = add(x = var_548, y = var_549)[name = tensor("filled_window_9")]; + tensor var_551 = greater_equal(x = filled_9, y = var_16)[name = tensor("op_551")]; + tensor var_552_dtype_0 = const()[name = tensor("op_552_dtype_0"), val = tensor("fp32")]; + tensor var_553 = const()[name = tensor("op_553"), val = tensor([1, 1, 1])]; + tensor var_552 = cast(dtype = var_552_dtype_0, x = var_551)[name = tensor("cast_166")]; + tensor use_roll_9 = reshape(shape = var_553, x = var_552)[name = tensor("use_roll_9")]; + tensor var_555 = sub(x = var_17, y = use_roll_9)[name = tensor("op_555")]; + tensor var_556 = mul(x = filled_window_9, y = var_555)[name = tensor("op_556")]; + tensor var_557 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_557")]; + tensor window_13 = add(x = var_556, y = var_557)[name = tensor("window_13")]; + tensor var_559 = const()[name = tensor("op_559"), val = tensor(0x1p+0)]; + tensor var_560 = add(x = filled_9, y = var_559)[name = tensor("op_560")]; + tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_560)[name = tensor("clip_9")]; + tensor var_563_begin_0 = const()[name = tensor("op_563_begin_0"), val = tensor([0, 1, 0])]; + tensor var_563_end_0 = const()[name = tensor("op_563_end_0"), val = tensor([1, 2, 256])]; + tensor var_563_end_mask_0 = const()[name = tensor("op_563_end_mask_0"), val = tensor([true, false, true])]; + tensor var_563 = slice_by_index(begin = var_563_begin_0, end = var_563_end_0, end_mask = var_563_end_mask_0, x = x_9)[name = tensor("op_563")]; + tensor var_566_begin_0 = const()[name = tensor("op_566_begin_0"), val = tensor([0, 1, 0])]; + tensor var_566_end_0 = const()[name = tensor("op_566_end_0"), val = tensor([1, 16, 256])]; + tensor var_566_end_mask_0 = const()[name = tensor("op_566_end_mask_0"), val = tensor([true, true, true])]; + tensor var_566 = slice_by_index(begin = var_566_begin_0, end = var_566_end_0, end_mask = var_566_end_mask_0, x = window_13)[name = tensor("op_566")]; tensor rolled_11_interleave_0 = const()[name = tensor("rolled_11_interleave_0"), val = tensor(false)]; - tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_781, var_778))[name = tensor("rolled_11")]; + tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_566, var_563))[name = tensor("rolled_11")]; tensor clip_10 = clip(alpha = var_21, beta = var_20, x = clip_9)[name = tensor("clip_10")]; - tensor var_786 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_786")]; - tensor var_787_dtype_0 = const()[name = tensor("op_787_dtype_0"), val = tensor("fp32")]; - tensor var_788 = const()[name = tensor("op_788"), val = tensor([1, 16, 1])]; - tensor var_787 = cast(dtype = var_787_dtype_0, x = var_786)[name = tensor("cast_165")]; - tensor fill_mask_11 = reshape(shape = var_788, x = var_787)[name = tensor("fill_mask_11")]; + tensor var_571 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_571")]; + tensor var_572_dtype_0 = const()[name = tensor("op_572_dtype_0"), val = tensor("fp32")]; + tensor var_573 = const()[name = tensor("op_573"), val = tensor([1, 16, 1])]; + tensor var_572 = cast(dtype = var_572_dtype_0, x = var_571)[name = tensor("cast_165")]; + tensor fill_mask_11 = reshape(shape = var_573, x = var_572)[name = tensor("fill_mask_11")]; tensor x_expanded_11_reps_0 = const()[name = tensor("x_expanded_11_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_778)[name = tensor("x_expanded_11")]; - tensor var_792 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_792")]; - tensor var_793 = mul(x = input_91, y = var_792)[name = tensor("op_793")]; - tensor var_794 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_794")]; - tensor filled_window_11 = add(x = var_793, y = var_794)[name = tensor("filled_window_11")]; - tensor var_796 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_796")]; - tensor var_797_dtype_0 = const()[name = tensor("op_797_dtype_0"), val = tensor("fp32")]; - tensor var_798 = const()[name = tensor("op_798"), val = tensor([1, 1, 1])]; - tensor var_797 = cast(dtype = var_797_dtype_0, x = var_796)[name = tensor("cast_164")]; - tensor use_roll_11 = reshape(shape = var_798, x = var_797)[name = tensor("use_roll_11")]; - tensor var_800 = sub(x = var_17, y = use_roll_11)[name = tensor("op_800")]; - tensor var_801 = mul(x = filled_window_11, y = var_800)[name = tensor("op_801")]; - tensor var_802 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_802")]; - tensor input_101 = add(x = var_801, y = var_802)[name = tensor("input_101")]; - tensor var_804 = const()[name = tensor("op_804"), val = tensor(0x1p+0)]; - tensor var_805 = add(x = clip_9, y = var_804)[name = tensor("op_805")]; - tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_805)[name = tensor("clip_11")]; - tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; - tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_101)[name = tensor("x_19")]; - tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_37_pad_type_0 = const()[name = tensor("inputs_37_pad_type_0"), val = tensor("valid")]; - tensor inputs_37_strides_0 = const()[name = tensor("inputs_37_strides_0"), val = tensor([1])]; - tensor inputs_37_pad_0 = const()[name = tensor("inputs_37_pad_0"), val = tensor([0, 0])]; - tensor inputs_37_dilations_0 = const()[name = tensor("inputs_37_dilations_0"), val = tensor([1])]; - tensor inputs_37_groups_0 = const()[name = tensor("inputs_37_groups_0"), val = tensor(1)]; - tensor input_103 = transpose(perm = input_103_perm_0, x = x_19)[name = tensor("transpose_60")]; - tensor inputs_37 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_37_dilations_0, groups = inputs_37_groups_0, pad = inputs_37_pad_0, pad_type = inputs_37_pad_type_0, strides = inputs_37_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_37")]; - tensor var_826_split_sizes_0 = const()[name = tensor("op_826_split_sizes_0"), val = tensor([256, 256])]; - tensor var_826_axis_0 = const()[name = tensor("op_826_axis_0"), val = tensor(1)]; - tensor var_826_0, tensor var_826_1 = split(axis = var_826_axis_0, split_sizes = var_826_split_sizes_0, x = inputs_37)[name = tensor("op_826")]; - tensor var_828 = sigmoid(x = var_826_1)[name = tensor("op_828")]; - tensor inputs_39 = mul(x = var_826_0, y = var_828)[name = tensor("inputs_39")]; - tensor outputs_aug_11_pad_type_0 = const()[name = tensor("outputs_aug_11_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_11_pad_0 = const()[name = tensor("outputs_aug_11_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_11_groups_0 = const()[name = tensor("outputs_aug_11_groups_0"), val = tensor(256)]; - tensor outputs_aug_11_strides_0 = const()[name = tensor("outputs_aug_11_strides_0"), val = tensor([1])]; - tensor outputs_aug_11_dilations_0 = const()[name = tensor("outputs_aug_11_dilations_0"), val = tensor([1])]; - tensor outputs_aug_11 = conv(dilations = outputs_aug_11_dilations_0, groups = outputs_aug_11_groups_0, pad = outputs_aug_11_pad_0, pad_type = outputs_aug_11_pad_type_0, strides = outputs_aug_11_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_39)[name = tensor("outputs_aug_11")]; - tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; - tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([1, 256, 16])]; - tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; - tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_11)[name = tensor("input_105")]; - tensor inputs_41 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_105)[name = tensor("inputs_41")]; - tensor input_107 = silu(x = inputs_41)[name = tensor("input_107")]; - tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; - tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; - tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; - tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; - tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; - tensor input_109 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; - tensor conv_out_11_perm_0 = const()[name = tensor("conv_out_11_perm_0"), val = tensor([0, 2, 1])]; - tensor var_859_begin_0 = const()[name = tensor("op_859_begin_0"), val = tensor([0, -1, 0])]; - tensor var_859_end_0 = const()[name = tensor("op_859_end_0"), val = tensor([1, 16, 256])]; - tensor var_859_end_mask_0 = const()[name = tensor("op_859_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_11 = transpose(perm = conv_out_11_perm_0, x = input_109)[name = tensor("transpose_59")]; - tensor var_859 = slice_by_index(begin = var_859_begin_0, end = var_859_end_0, end_mask = var_859_end_mask_0, x = conv_out_11)[name = tensor("op_859")]; - tensor var_862_begin_0 = const()[name = tensor("op_862_begin_0"), val = tensor([0, 2, 0])]; - tensor var_862_end_0 = const()[name = tensor("op_862_end_0"), val = tensor([1, 3, 256])]; - tensor var_862_end_mask_0 = const()[name = tensor("op_862_end_mask_0"), val = tensor([true, false, true])]; - tensor var_862 = slice_by_index(begin = var_862_begin_0, end = var_862_end_0, end_mask = var_862_end_mask_0, x = x_15)[name = tensor("op_862")]; - tensor var_865_begin_0 = const()[name = tensor("op_865_begin_0"), val = tensor([0, 1, 0])]; - tensor var_865_end_0 = const()[name = tensor("op_865_end_0"), val = tensor([1, 16, 256])]; - tensor var_865_end_mask_0 = const()[name = tensor("op_865_end_mask_0"), val = tensor([true, true, true])]; - tensor var_865 = slice_by_index(begin = var_865_begin_0, end = var_865_end_0, end_mask = var_865_end_mask_0, x = input_101)[name = tensor("op_865")]; + tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_563)[name = tensor("x_expanded_11")]; + tensor var_577 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_577")]; + tensor var_578 = mul(x = window_13, y = var_577)[name = tensor("op_578")]; + tensor var_579 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_579")]; + tensor filled_window_11 = add(x = var_578, y = var_579)[name = tensor("filled_window_11")]; + tensor var_581 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_581")]; + tensor var_582_dtype_0 = const()[name = tensor("op_582_dtype_0"), val = tensor("fp32")]; + tensor var_583 = const()[name = tensor("op_583"), val = tensor([1, 1, 1])]; + tensor var_582 = cast(dtype = var_582_dtype_0, x = var_581)[name = tensor("cast_164")]; + tensor use_roll_11 = reshape(shape = var_583, x = var_582)[name = tensor("use_roll_11")]; + tensor var_585 = sub(x = var_17, y = use_roll_11)[name = tensor("op_585")]; + tensor var_586 = mul(x = filled_window_11, y = var_585)[name = tensor("op_586")]; + tensor var_587 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_587")]; + tensor window_15 = add(x = var_586, y = var_587)[name = tensor("window_15")]; + tensor var_589 = const()[name = tensor("op_589"), val = tensor(0x1p+0)]; + tensor var_590 = add(x = clip_9, y = var_589)[name = tensor("op_590")]; + tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_590)[name = tensor("clip_11")]; + tensor var_593_begin_0 = const()[name = tensor("op_593_begin_0"), val = tensor([0, 2, 0])]; + tensor var_593_end_0 = const()[name = tensor("op_593_end_0"), val = tensor([1, 3, 256])]; + tensor var_593_end_mask_0 = const()[name = tensor("op_593_end_mask_0"), val = tensor([true, false, true])]; + tensor var_593 = slice_by_index(begin = var_593_begin_0, end = var_593_end_0, end_mask = var_593_end_mask_0, x = x_9)[name = tensor("op_593")]; + tensor var_596_begin_0 = const()[name = tensor("op_596_begin_0"), val = tensor([0, 1, 0])]; + tensor var_596_end_0 = const()[name = tensor("op_596_end_0"), val = tensor([1, 16, 256])]; + tensor var_596_end_mask_0 = const()[name = tensor("op_596_end_mask_0"), val = tensor([true, true, true])]; + tensor var_596 = slice_by_index(begin = var_596_begin_0, end = var_596_end_0, end_mask = var_596_end_mask_0, x = window_15)[name = tensor("op_596")]; tensor rolled_13_interleave_0 = const()[name = tensor("rolled_13_interleave_0"), val = tensor(false)]; - tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_865, var_862))[name = tensor("rolled_13")]; + tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_596, var_593))[name = tensor("rolled_13")]; tensor clip_12 = clip(alpha = var_21, beta = var_20, x = clip_11)[name = tensor("clip_12")]; - tensor var_870 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_870")]; - tensor var_871_dtype_0 = const()[name = tensor("op_871_dtype_0"), val = tensor("fp32")]; - tensor var_872 = const()[name = tensor("op_872"), val = tensor([1, 16, 1])]; - tensor var_871 = cast(dtype = var_871_dtype_0, x = var_870)[name = tensor("cast_163")]; - tensor fill_mask_13 = reshape(shape = var_872, x = var_871)[name = tensor("fill_mask_13")]; + tensor var_601 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_601")]; + tensor var_602_dtype_0 = const()[name = tensor("op_602_dtype_0"), val = tensor("fp32")]; + tensor var_603 = const()[name = tensor("op_603"), val = tensor([1, 16, 1])]; + tensor var_602 = cast(dtype = var_602_dtype_0, x = var_601)[name = tensor("cast_163")]; + tensor fill_mask_13 = reshape(shape = var_603, x = var_602)[name = tensor("fill_mask_13")]; tensor x_expanded_13_reps_0 = const()[name = tensor("x_expanded_13_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_862)[name = tensor("x_expanded_13")]; - tensor var_876 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_876")]; - tensor var_877 = mul(x = input_101, y = var_876)[name = tensor("op_877")]; - tensor var_878 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_878")]; - tensor filled_window_13 = add(x = var_877, y = var_878)[name = tensor("filled_window_13")]; - tensor var_880 = greater_equal(x = clip_11, y = var_16)[name = tensor("op_880")]; - tensor var_881_dtype_0 = const()[name = tensor("op_881_dtype_0"), val = tensor("fp32")]; - tensor var_882 = const()[name = tensor("op_882"), val = tensor([1, 1, 1])]; - tensor var_881 = cast(dtype = var_881_dtype_0, x = var_880)[name = tensor("cast_162")]; - tensor use_roll_13 = reshape(shape = var_882, x = var_881)[name = tensor("use_roll_13")]; - tensor var_884 = sub(x = var_17, y = use_roll_13)[name = tensor("op_884")]; - tensor var_885 = mul(x = filled_window_13, y = var_884)[name = tensor("op_885")]; - tensor var_886 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_886")]; - tensor input_111 = add(x = var_885, y = var_886)[name = tensor("input_111")]; - tensor var_888 = const()[name = tensor("op_888"), val = tensor(0x1p+0)]; - tensor var_889 = add(x = clip_11, y = var_888)[name = tensor("op_889")]; - tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_889)[name = tensor("clip_13")]; - tensor x_21_axes_0 = const()[name = tensor("x_21_axes_0"), val = tensor([-1])]; - tensor x_21 = layer_norm(axes = x_21_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_111)[name = tensor("x_21")]; - tensor input_113_perm_0 = const()[name = tensor("input_113_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_43_pad_type_0 = const()[name = tensor("inputs_43_pad_type_0"), val = tensor("valid")]; - tensor inputs_43_strides_0 = const()[name = tensor("inputs_43_strides_0"), val = tensor([1])]; - tensor inputs_43_pad_0 = const()[name = tensor("inputs_43_pad_0"), val = tensor([0, 0])]; - tensor inputs_43_dilations_0 = const()[name = tensor("inputs_43_dilations_0"), val = tensor([1])]; - tensor inputs_43_groups_0 = const()[name = tensor("inputs_43_groups_0"), val = tensor(1)]; - tensor input_113 = transpose(perm = input_113_perm_0, x = x_21)[name = tensor("transpose_58")]; - tensor inputs_43 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_43_dilations_0, groups = inputs_43_groups_0, pad = inputs_43_pad_0, pad_type = inputs_43_pad_type_0, strides = inputs_43_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_113)[name = tensor("inputs_43")]; - tensor var_910_split_sizes_0 = const()[name = tensor("op_910_split_sizes_0"), val = tensor([256, 256])]; - tensor var_910_axis_0 = const()[name = tensor("op_910_axis_0"), val = tensor(1)]; - tensor var_910_0, tensor var_910_1 = split(axis = var_910_axis_0, split_sizes = var_910_split_sizes_0, x = inputs_43)[name = tensor("op_910")]; - tensor var_912 = sigmoid(x = var_910_1)[name = tensor("op_912")]; - tensor inputs_45 = mul(x = var_910_0, y = var_912)[name = tensor("inputs_45")]; - tensor outputs_aug_13_pad_type_0 = const()[name = tensor("outputs_aug_13_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_13_pad_0 = const()[name = tensor("outputs_aug_13_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_13_groups_0 = const()[name = tensor("outputs_aug_13_groups_0"), val = tensor(256)]; - tensor outputs_aug_13_strides_0 = const()[name = tensor("outputs_aug_13_strides_0"), val = tensor([1])]; - tensor outputs_aug_13_dilations_0 = const()[name = tensor("outputs_aug_13_dilations_0"), val = tensor([1])]; - tensor outputs_aug_13 = conv(dilations = outputs_aug_13_dilations_0, groups = outputs_aug_13_groups_0, pad = outputs_aug_13_pad_0, pad_type = outputs_aug_13_pad_type_0, strides = outputs_aug_13_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_45)[name = tensor("outputs_aug_13")]; - tensor input_115_begin_0 = const()[name = tensor("input_115_begin_0"), val = tensor([0, 0, 0])]; - tensor input_115_end_0 = const()[name = tensor("input_115_end_0"), val = tensor([1, 256, 16])]; - tensor input_115_end_mask_0 = const()[name = tensor("input_115_end_mask_0"), val = tensor([true, true, false])]; - tensor input_115 = slice_by_index(begin = input_115_begin_0, end = input_115_end_0, end_mask = input_115_end_mask_0, x = outputs_aug_13)[name = tensor("input_115")]; - tensor inputs_47 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_115)[name = tensor("inputs_47")]; - tensor input_117 = silu(x = inputs_47)[name = tensor("input_117")]; - tensor input_119_pad_type_0 = const()[name = tensor("input_119_pad_type_0"), val = tensor("valid")]; - tensor input_119_strides_0 = const()[name = tensor("input_119_strides_0"), val = tensor([1])]; - tensor input_119_pad_0 = const()[name = tensor("input_119_pad_0"), val = tensor([0, 0])]; - tensor input_119_dilations_0 = const()[name = tensor("input_119_dilations_0"), val = tensor([1])]; - tensor input_119_groups_0 = const()[name = tensor("input_119_groups_0"), val = tensor(1)]; - tensor input_119 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_119_dilations_0, groups = input_119_groups_0, pad = input_119_pad_0, pad_type = input_119_pad_type_0, strides = input_119_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_117)[name = tensor("input_119")]; - tensor conv_out_13_perm_0 = const()[name = tensor("conv_out_13_perm_0"), val = tensor([0, 2, 1])]; - tensor var_943_begin_0 = const()[name = tensor("op_943_begin_0"), val = tensor([0, -1, 0])]; - tensor var_943_end_0 = const()[name = tensor("op_943_end_0"), val = tensor([1, 16, 256])]; - tensor var_943_end_mask_0 = const()[name = tensor("op_943_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_13 = transpose(perm = conv_out_13_perm_0, x = input_119)[name = tensor("transpose_57")]; - tensor var_943 = slice_by_index(begin = var_943_begin_0, end = var_943_end_0, end_mask = var_943_end_mask_0, x = conv_out_13)[name = tensor("op_943")]; - tensor var_946_begin_0 = const()[name = tensor("op_946_begin_0"), val = tensor([0, 3, 0])]; - tensor var_946_end_0 = const()[name = tensor("op_946_end_0"), val = tensor([1, 1, 256])]; - tensor var_946_end_mask_0 = const()[name = tensor("op_946_end_mask_0"), val = tensor([true, true, true])]; - tensor var_946 = slice_by_index(begin = var_946_begin_0, end = var_946_end_0, end_mask = var_946_end_mask_0, x = x_15)[name = tensor("op_946")]; - tensor var_949_begin_0 = const()[name = tensor("op_949_begin_0"), val = tensor([0, 1, 0])]; - tensor var_949_end_0 = const()[name = tensor("op_949_end_0"), val = tensor([1, 16, 256])]; - tensor var_949_end_mask_0 = const()[name = tensor("op_949_end_mask_0"), val = tensor([true, true, true])]; - tensor var_949 = slice_by_index(begin = var_949_begin_0, end = var_949_end_0, end_mask = var_949_end_mask_0, x = input_111)[name = tensor("op_949")]; + tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_593)[name = tensor("x_expanded_13")]; + tensor var_607 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_607")]; + tensor var_608 = mul(x = window_15, y = var_607)[name = tensor("op_608")]; + tensor var_609 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_609")]; + tensor filled_window_13 = add(x = var_608, y = var_609)[name = tensor("filled_window_13")]; + tensor var_611 = greater_equal(x = clip_11, y = var_16)[name = tensor("op_611")]; + tensor var_612_dtype_0 = const()[name = tensor("op_612_dtype_0"), val = tensor("fp32")]; + tensor var_613 = const()[name = tensor("op_613"), val = tensor([1, 1, 1])]; + tensor var_612 = cast(dtype = var_612_dtype_0, x = var_611)[name = tensor("cast_162")]; + tensor use_roll_13 = reshape(shape = var_613, x = var_612)[name = tensor("use_roll_13")]; + tensor var_615 = sub(x = var_17, y = use_roll_13)[name = tensor("op_615")]; + tensor var_616 = mul(x = filled_window_13, y = var_615)[name = tensor("op_616")]; + tensor var_617 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_617")]; + tensor window_17 = add(x = var_616, y = var_617)[name = tensor("window_17")]; + tensor var_619 = const()[name = tensor("op_619"), val = tensor(0x1p+0)]; + tensor var_620 = add(x = clip_11, y = var_619)[name = tensor("op_620")]; + tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_620)[name = tensor("clip_13")]; + tensor var_623_begin_0 = const()[name = tensor("op_623_begin_0"), val = tensor([0, 3, 0])]; + tensor var_623_end_0 = const()[name = tensor("op_623_end_0"), val = tensor([1, 1, 256])]; + tensor var_623_end_mask_0 = const()[name = tensor("op_623_end_mask_0"), val = tensor([true, true, true])]; + tensor var_623 = slice_by_index(begin = var_623_begin_0, end = var_623_end_0, end_mask = var_623_end_mask_0, x = x_9)[name = tensor("op_623")]; + tensor var_626_begin_0 = const()[name = tensor("op_626_begin_0"), val = tensor([0, 1, 0])]; + tensor var_626_end_0 = const()[name = tensor("op_626_end_0"), val = tensor([1, 16, 256])]; + tensor var_626_end_mask_0 = const()[name = tensor("op_626_end_mask_0"), val = tensor([true, true, true])]; + tensor var_626 = slice_by_index(begin = var_626_begin_0, end = var_626_end_0, end_mask = var_626_end_mask_0, x = window_17)[name = tensor("op_626")]; tensor rolled_15_interleave_0 = const()[name = tensor("rolled_15_interleave_0"), val = tensor(false)]; - tensor rolled_15 = concat(axis = var_33, interleave = rolled_15_interleave_0, values = (var_949, var_946))[name = tensor("rolled_15")]; + tensor rolled_15 = concat(axis = var_33, interleave = rolled_15_interleave_0, values = (var_626, var_623))[name = tensor("rolled_15")]; tensor clip_14 = clip(alpha = var_21, beta = var_20, x = clip_13)[name = tensor("clip_14")]; - tensor var_954 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_954")]; - tensor var_955_dtype_0 = const()[name = tensor("op_955_dtype_0"), val = tensor("fp32")]; - tensor var_956 = const()[name = tensor("op_956"), val = tensor([1, 16, 1])]; - tensor var_955 = cast(dtype = var_955_dtype_0, x = var_954)[name = tensor("cast_161")]; - tensor fill_mask_15 = reshape(shape = var_956, x = var_955)[name = tensor("fill_mask_15")]; + tensor var_631 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_631")]; + tensor var_632_dtype_0 = const()[name = tensor("op_632_dtype_0"), val = tensor("fp32")]; + tensor var_633 = const()[name = tensor("op_633"), val = tensor([1, 16, 1])]; + tensor var_632 = cast(dtype = var_632_dtype_0, x = var_631)[name = tensor("cast_161")]; + tensor fill_mask_15 = reshape(shape = var_633, x = var_632)[name = tensor("fill_mask_15")]; tensor x_expanded_15_reps_0 = const()[name = tensor("x_expanded_15_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_946)[name = tensor("x_expanded_15")]; - tensor var_960 = sub(x = var_17, y = fill_mask_15)[name = tensor("op_960")]; - tensor var_961 = mul(x = input_111, y = var_960)[name = tensor("op_961")]; - tensor var_962 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_962")]; - tensor filled_window_15 = add(x = var_961, y = var_962)[name = tensor("filled_window_15")]; - tensor var_964 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_964")]; - tensor var_965_dtype_0 = const()[name = tensor("op_965_dtype_0"), val = tensor("fp32")]; - tensor var_966 = const()[name = tensor("op_966"), val = tensor([1, 1, 1])]; - tensor var_965 = cast(dtype = var_965_dtype_0, x = var_964)[name = tensor("cast_160")]; - tensor use_roll_15 = reshape(shape = var_966, x = var_965)[name = tensor("use_roll_15")]; - tensor var_968 = sub(x = var_17, y = use_roll_15)[name = tensor("op_968")]; - tensor var_969 = mul(x = filled_window_15, y = var_968)[name = tensor("op_969")]; - tensor var_970 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_970")]; - tensor input_121 = add(x = var_969, y = var_970)[name = tensor("input_121")]; - tensor var_972 = const()[name = tensor("op_972"), val = tensor(0x1p+0)]; - tensor var_973 = add(x = clip_13, y = var_972)[name = tensor("op_973")]; - tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_973)[name = tensor("clip_15")]; - tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; - tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_121)[name = tensor("x_23")]; - tensor input_123_perm_0 = const()[name = tensor("input_123_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_49_pad_type_0 = const()[name = tensor("inputs_49_pad_type_0"), val = tensor("valid")]; - tensor inputs_49_strides_0 = const()[name = tensor("inputs_49_strides_0"), val = tensor([1])]; - tensor inputs_49_pad_0 = const()[name = tensor("inputs_49_pad_0"), val = tensor([0, 0])]; - tensor inputs_49_dilations_0 = const()[name = tensor("inputs_49_dilations_0"), val = tensor([1])]; - tensor inputs_49_groups_0 = const()[name = tensor("inputs_49_groups_0"), val = tensor(1)]; - tensor input_123 = transpose(perm = input_123_perm_0, x = x_23)[name = tensor("transpose_56")]; - tensor inputs_49 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_49_dilations_0, groups = inputs_49_groups_0, pad = inputs_49_pad_0, pad_type = inputs_49_pad_type_0, strides = inputs_49_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_123)[name = tensor("inputs_49")]; - tensor var_994_split_sizes_0 = const()[name = tensor("op_994_split_sizes_0"), val = tensor([256, 256])]; - tensor var_994_axis_0 = const()[name = tensor("op_994_axis_0"), val = tensor(1)]; - tensor var_994_0, tensor var_994_1 = split(axis = var_994_axis_0, split_sizes = var_994_split_sizes_0, x = inputs_49)[name = tensor("op_994")]; - tensor var_996 = sigmoid(x = var_994_1)[name = tensor("op_996")]; - tensor inputs_51 = mul(x = var_994_0, y = var_996)[name = tensor("inputs_51")]; - tensor outputs_aug_15_pad_type_0 = const()[name = tensor("outputs_aug_15_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_15_pad_0 = const()[name = tensor("outputs_aug_15_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_15_groups_0 = const()[name = tensor("outputs_aug_15_groups_0"), val = tensor(256)]; - tensor outputs_aug_15_strides_0 = const()[name = tensor("outputs_aug_15_strides_0"), val = tensor([1])]; - tensor outputs_aug_15_dilations_0 = const()[name = tensor("outputs_aug_15_dilations_0"), val = tensor([1])]; - tensor outputs_aug_15 = conv(dilations = outputs_aug_15_dilations_0, groups = outputs_aug_15_groups_0, pad = outputs_aug_15_pad_0, pad_type = outputs_aug_15_pad_type_0, strides = outputs_aug_15_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_51)[name = tensor("outputs_aug_15")]; - tensor input_125_begin_0 = const()[name = tensor("input_125_begin_0"), val = tensor([0, 0, 0])]; - tensor input_125_end_0 = const()[name = tensor("input_125_end_0"), val = tensor([1, 256, 16])]; - tensor input_125_end_mask_0 = const()[name = tensor("input_125_end_mask_0"), val = tensor([true, true, false])]; - tensor input_125 = slice_by_index(begin = input_125_begin_0, end = input_125_end_0, end_mask = input_125_end_mask_0, x = outputs_aug_15)[name = tensor("input_125")]; - tensor inputs_53 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_125)[name = tensor("inputs_53")]; - tensor input_127 = silu(x = inputs_53)[name = tensor("input_127")]; - tensor input_129_pad_type_0 = const()[name = tensor("input_129_pad_type_0"), val = tensor("valid")]; - tensor input_129_strides_0 = const()[name = tensor("input_129_strides_0"), val = tensor([1])]; - tensor input_129_pad_0 = const()[name = tensor("input_129_pad_0"), val = tensor([0, 0])]; - tensor input_129_dilations_0 = const()[name = tensor("input_129_dilations_0"), val = tensor([1])]; - tensor input_129_groups_0 = const()[name = tensor("input_129_groups_0"), val = tensor(1)]; - tensor input_129 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_129_dilations_0, groups = input_129_groups_0, pad = input_129_pad_0, pad_type = input_129_pad_type_0, strides = input_129_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_127)[name = tensor("input_129")]; - tensor conv_out_15_perm_0 = const()[name = tensor("conv_out_15_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1027_begin_0 = const()[name = tensor("op_1027_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1027_end_0 = const()[name = tensor("op_1027_end_0"), val = tensor([1, 16, 256])]; - tensor var_1027_end_mask_0 = const()[name = tensor("op_1027_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_15 = transpose(perm = conv_out_15_perm_0, x = input_129)[name = tensor("transpose_55")]; - tensor var_1027 = slice_by_index(begin = var_1027_begin_0, end = var_1027_end_0, end_mask = var_1027_end_mask_0, x = conv_out_15)[name = tensor("op_1027")]; - tensor var_1030_interleave_0 = const()[name = tensor("op_1030_interleave_0"), val = tensor(false)]; - tensor var_1030 = concat(axis = var_33, interleave = var_1030_interleave_0, values = (var_775, var_859, var_943, var_1027))[name = tensor("op_1030")]; - tensor input_131 = add(x = x_15, y = var_1030)[name = tensor("input_131")]; - tensor input_133_axes_0 = const()[name = tensor("input_133_axes_0"), val = tensor([-1])]; - tensor input_133 = layer_norm(axes = input_133_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_131)[name = tensor("input_133")]; - tensor inputs_55 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_133)[name = tensor("linear_17")]; - tensor input_135 = silu(x = inputs_55)[name = tensor("input_135")]; - tensor input_139 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_135)[name = tensor("linear_18")]; - tensor var_1053 = const()[name = tensor("op_1053"), val = tensor(0x1p-1)]; - tensor var_1054 = mul(x = input_139, y = var_1053)[name = tensor("op_1054")]; - tensor input_141 = add(x = var_1054, y = input_131)[name = tensor("input_141")]; - tensor input_143_axes_0 = const()[name = tensor("input_143_axes_0"), val = tensor([-1])]; - tensor input_143 = layer_norm(axes = input_143_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_141)[name = tensor("input_143")]; - tensor input_145_axes_0 = const()[name = tensor("input_145_axes_0"), val = tensor([-1])]; - tensor input_145 = layer_norm(axes = input_145_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_143)[name = tensor("input_145")]; - tensor inputs_57 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_145)[name = tensor("linear_19")]; - tensor input_147 = silu(x = inputs_57)[name = tensor("input_147")]; - tensor input_151 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_147)[name = tensor("linear_20")]; - tensor var_1083 = const()[name = tensor("op_1083"), val = tensor(0x1p-1)]; - tensor var_1084 = mul(x = input_151, y = var_1083)[name = tensor("op_1084")]; - tensor input_153 = add(x = var_1084, y = input_143)[name = tensor("input_153")]; - tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; - tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_153)[name = tensor("x_25")]; + tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_623)[name = tensor("x_expanded_15")]; + tensor var_637 = sub(x = var_17, y = fill_mask_15)[name = tensor("op_637")]; + tensor var_638 = mul(x = window_17, y = var_637)[name = tensor("op_638")]; + tensor var_639 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_639")]; + tensor filled_window_15 = add(x = var_638, y = var_639)[name = tensor("filled_window_15")]; + tensor var_641 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_641")]; + tensor var_642_dtype_0 = const()[name = tensor("op_642_dtype_0"), val = tensor("fp32")]; + tensor var_643 = const()[name = tensor("op_643"), val = tensor([1, 1, 1])]; + tensor var_642 = cast(dtype = var_642_dtype_0, x = var_641)[name = tensor("cast_160")]; + tensor use_roll_15 = reshape(shape = var_643, x = var_642)[name = tensor("use_roll_15")]; + tensor var_645 = sub(x = var_17, y = use_roll_15)[name = tensor("op_645")]; + tensor var_646 = mul(x = filled_window_15, y = var_645)[name = tensor("op_646")]; + tensor var_647 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_647")]; + tensor window_19 = add(x = var_646, y = var_647)[name = tensor("window_19")]; + tensor var_649 = const()[name = tensor("op_649"), val = tensor(0x1p+0)]; + tensor var_650 = add(x = clip_13, y = var_649)[name = tensor("op_650")]; + tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_650)[name = tensor("clip_15")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_30, interleave = input_61_interleave_0, values = (window_13, window_15, window_17, window_19))[name = tensor("input_61")]; + tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; + tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; + tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_13_pad_type_0 = const()[name = tensor("inputs_13_pad_type_0"), val = tensor("valid")]; + tensor inputs_13_strides_0 = const()[name = tensor("inputs_13_strides_0"), val = tensor([1])]; + tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; + tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; + tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; + tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; + tensor var_673_split_sizes_0 = const()[name = tensor("op_673_split_sizes_0"), val = tensor([256, 256])]; + tensor var_673_axis_0 = const()[name = tensor("op_673_axis_0"), val = tensor(1)]; + tensor var_673_0, tensor var_673_1 = split(axis = var_673_axis_0, split_sizes = var_673_split_sizes_0, x = inputs_13)[name = tensor("op_673")]; + tensor var_675 = sigmoid(x = var_673_1)[name = tensor("op_675")]; + tensor inputs_15 = mul(x = var_673_0, y = var_675)[name = tensor("inputs_15")]; + tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; + tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; + tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; + tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_15)[name = tensor("outputs_aug_3")]; + tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; + tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([4, 256, 16])]; + tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; + tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_3)[name = tensor("input_65")]; + tensor inputs_17 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_65)[name = tensor("inputs_17")]; + tensor input_67 = silu(x = inputs_17)[name = tensor("input_67")]; + tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; + tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; + tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; + tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; + tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; + tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; + tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; + tensor var_706_begin_0 = const()[name = tensor("op_706_begin_0"), val = tensor([0, -1, 0])]; + tensor var_706_end_0 = const()[name = tensor("op_706_end_0"), val = tensor([4, 16, 256])]; + tensor var_706_end_mask_0 = const()[name = tensor("op_706_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_706 = slice_by_index(begin = var_706_begin_0, end = var_706_end_0, end_mask = var_706_end_mask_0, x = conv_out_3)[name = tensor("op_706")]; + tensor var_708_perm_0 = const()[name = tensor("op_708_perm_0"), val = tensor([1, 0, 2])]; + tensor var_708 = transpose(perm = var_708_perm_0, x = var_706)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_708)[name = tensor("input_71")]; + tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; + tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; + tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; + tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; + tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; + tensor var_731 = const()[name = tensor("op_731"), val = tensor(0x1p-1)]; + tensor var_732 = mul(x = input_79, y = var_731)[name = tensor("op_732")]; + tensor input_81 = add(x = var_732, y = input_71)[name = tensor("input_81")]; + tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; + tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; + tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; + tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; + tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; + tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; + tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; + tensor var_761 = const()[name = tensor("op_761"), val = tensor(0x1p-1)]; + tensor var_762 = mul(x = input_91, y = var_761)[name = tensor("op_762")]; + tensor input_93 = add(x = var_762, y = input_83)[name = tensor("input_93")]; + tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; + tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; tensor prev_kv_5_end_0 = const()[name = tensor("prev_kv_5_end_0"), val = tensor([3, 1, 4, 64, 64])]; tensor prev_kv_5_end_mask_0 = const()[name = tensor("prev_kv_5_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1029,389 +800,274 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_1098 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_25)[name = tensor("linear_21")]; - tensor var_1099 = const()[name = tensor("op_1099"), val = tensor([1, 4, 4, 64])]; - tensor var_1100 = reshape(shape = var_1099, x = var_1098)[name = tensor("op_1100")]; + tensor var_776 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_777 = const()[name = tensor("op_777"), val = tensor([1, 4, 4, 64])]; + tensor var_778 = reshape(shape = var_777, x = var_776)[name = tensor("op_778")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1104 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_25)[name = tensor("linear_22")]; - tensor var_1105 = const()[name = tensor("op_1105"), val = tensor(0x1p-3)]; - tensor var_1106 = mul(x = var_1104, y = var_1105)[name = tensor("op_1106")]; - tensor var_1107 = const()[name = tensor("op_1107"), val = tensor([1, 4, 4, 64])]; - tensor var_1108 = reshape(shape = var_1107, x = var_1106)[name = tensor("op_1108")]; + tensor var_782 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_783 = const()[name = tensor("op_783"), val = tensor(0x1p-3)]; + tensor var_784 = mul(x = var_782, y = var_783)[name = tensor("op_784")]; + tensor var_785 = const()[name = tensor("op_785"), val = tensor([1, 4, 4, 64])]; + tensor var_786 = reshape(shape = var_785, x = var_784)[name = tensor("op_786")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1112 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_25)[name = tensor("linear_23")]; - tensor var_1113 = const()[name = tensor("op_1113"), val = tensor([1, 4, 4, 64])]; - tensor var_1114 = reshape(shape = var_1113, x = var_1112)[name = tensor("op_1114")]; + tensor var_790 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_791 = const()[name = tensor("op_791"), val = tensor([1, 4, 4, 64])]; + tensor var_792 = reshape(shape = var_791, x = var_790)[name = tensor("op_792")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_157 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_25)[name = tensor("linear_24")]; + tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; tensor s_t_5 = add(x = prev_scale_5, y = encoder__t_index)[name = tensor("s_t_5")]; tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_1108)[name = tensor("transpose_53")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_1100)[name = tensor("transpose_54")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_786)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_778)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_1124 = const()[name = tensor("op_1124"), val = tensor([4, 1])]; - tensor var_1125 = reshape(shape = var_1124, x = sqrt_s_t_5)[name = tensor("op_1125")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_1125)[name = tensor("M_5")]; - tensor var_1127 = mul(x = qk_5, y = M_5)[name = tensor("op_1127")]; + tensor var_802 = const()[name = tensor("op_802"), val = tensor([4, 1])]; + tensor var_803 = reshape(shape = var_802, x = sqrt_s_t_5)[name = tensor("op_803")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_803)[name = tensor("M_5")]; + tensor var_805 = mul(x = qk_5, y = M_5)[name = tensor("op_805")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_1114)[name = tensor("transpose_52")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_1127, y = v_5)[name = tensor("inner_5")]; - tensor var_1129_transpose_x_0 = const()[name = tensor("op_1129_transpose_x_0"), val = tensor(false)]; - tensor var_1129_transpose_y_0 = const()[name = tensor("op_1129_transpose_y_0"), val = tensor(false)]; - tensor var_1129 = matmul(transpose_x = var_1129_transpose_x_0, transpose_y = var_1129_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_1129")]; - tensor var_1130 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_1130")]; - tensor var_1131 = const()[name = tensor("op_1131"), val = tensor([1, 1, 4, 1])]; - tensor var_1132 = reshape(shape = var_1131, x = var_1130)[name = tensor("op_1132")]; - tensor cross_5 = mul(x = var_1129, y = var_1132)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_792)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_805, y = v_5)[name = tensor("inner_5")]; + tensor var_807_transpose_x_0 = const()[name = tensor("op_807_transpose_x_0"), val = tensor(false)]; + tensor var_807_transpose_y_0 = const()[name = tensor("op_807_transpose_y_0"), val = tensor(false)]; + tensor var_807 = matmul(transpose_x = var_807_transpose_x_0, transpose_y = var_807_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_807")]; + tensor var_808 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_808")]; + tensor var_809 = const()[name = tensor("op_809"), val = tensor([1, 1, 4, 1])]; + tensor var_810 = reshape(shape = var_809, x = var_808)[name = tensor("op_810")]; + tensor cross_5 = mul(x = var_807, y = var_810)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_1135 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_1135")]; - tensor var_1137_transpose_x_1 = const()[name = tensor("op_1137_transpose_x_1"), val = tensor(true)]; - tensor var_1137_transpose_y_1 = const()[name = tensor("op_1137_transpose_y_1"), val = tensor(false)]; - tensor var_1137 = matmul(transpose_x = var_1137_transpose_x_1, transpose_y = var_1137_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_1137")]; - tensor new_kv_unnorm_5 = add(x = var_1135, y = var_1137)[name = tensor("new_kv_unnorm_5")]; - tensor var_1139 = const()[name = tensor("op_1139"), val = tensor(0x1p+2)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_1139)[name = tensor("new_scale_5")]; - tensor var_1141 = sqrt(x = new_scale_5)[name = tensor("op_1141")]; - tensor var_1142 = real_div(x = new_kv_unnorm_5, y = var_1141)[name = tensor("op_1142")]; - tensor var_1143_perm_0 = const()[name = tensor("op_1143_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_813 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_813")]; + tensor var_815_transpose_x_1 = const()[name = tensor("op_815_transpose_x_1"), val = tensor(true)]; + tensor var_815_transpose_y_1 = const()[name = tensor("op_815_transpose_y_1"), val = tensor(false)]; + tensor var_815 = matmul(transpose_x = var_815_transpose_x_1, transpose_y = var_815_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_815")]; + tensor new_kv_unnorm_5 = add(x = var_813, y = var_815)[name = tensor("new_kv_unnorm_5")]; + tensor var_817 = const()[name = tensor("op_817"), val = tensor(0x1p+2)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_817)[name = tensor("new_scale_5")]; + tensor var_819 = sqrt(x = new_scale_5)[name = tensor("op_819")]; + tensor var_820 = real_div(x = new_kv_unnorm_5, y = var_819)[name = tensor("op_820")]; + tensor var_821_perm_0 = const()[name = tensor("op_821_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_1143 = transpose(perm = var_1143_perm_0, x = out_13)[name = tensor("transpose_51")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_1143)[name = tensor("out_15")]; - tensor var_1147 = const()[name = tensor("op_1147"), val = tensor([1, 4, 256])]; - tensor out_17 = reshape(shape = var_1147, x = out_15)[name = tensor("out_17")]; - tensor var_1149 = silu(x = input_157)[name = tensor("op_1149")]; - tensor input_159 = mul(x = var_1149, y = out_17)[name = tensor("input_159")]; - tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_159)[name = tensor("linear_25")]; - tensor x_27 = add(x = input_153, y = ret_out_5)[name = tensor("x_27")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor var_821 = transpose(perm = var_821_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_821)[name = tensor("out_15")]; + tensor var_825 = const()[name = tensor("op_825"), val = tensor([1, 4, 256])]; + tensor out_17 = reshape(shape = var_825, x = out_15)[name = tensor("out_17")]; + tensor var_827 = silu(x = input_97)[name = tensor("op_827")]; + tensor input_99 = mul(x = var_827, y = out_17)[name = tensor("input_99")]; + tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; + tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; + tensor window_21_begin_0 = const()[name = tensor("window_21_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_21_end_0 = const()[name = tensor("window_21_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_21_end_mask_0 = const()[name = tensor("window_21_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_21_squeeze_mask_0 = const()[name = tensor("window_21_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_21 = slice_by_index(begin = window_21_begin_0, end = window_21_end_0, end_mask = window_21_end_mask_0, squeeze_mask = window_21_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_21")]; tensor filled_17_begin_0 = const()[name = tensor("filled_17_begin_0"), val = tensor([2, 0])]; tensor filled_17_end_0 = const()[name = tensor("filled_17_end_0"), val = tensor([3, 1])]; tensor filled_17_end_mask_0 = const()[name = tensor("filled_17_end_mask_0"), val = tensor([false, true])]; tensor filled_17_squeeze_mask_0 = const()[name = tensor("filled_17_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_17 = slice_by_index(begin = filled_17_begin_0, end = filled_17_end_0, end_mask = filled_17_end_mask_0, squeeze_mask = filled_17_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_17")]; - tensor var_1158_begin_0 = const()[name = tensor("op_1158_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1158_end_0 = const()[name = tensor("op_1158_end_0"), val = tensor([1, 1, 256])]; - tensor var_1158_end_mask_0 = const()[name = tensor("op_1158_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1158 = slice_by_index(begin = var_1158_begin_0, end = var_1158_end_0, end_mask = var_1158_end_mask_0, x = x_27)[name = tensor("op_1158")]; - tensor var_1161_begin_0 = const()[name = tensor("op_1161_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1161_end_0 = const()[name = tensor("op_1161_end_0"), val = tensor([1, 16, 256])]; - tensor var_1161_end_mask_0 = const()[name = tensor("op_1161_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1161 = slice_by_index(begin = var_1161_begin_0, end = var_1161_end_0, end_mask = var_1161_end_mask_0, x = window_5)[name = tensor("op_1161")]; + tensor var_836_begin_0 = const()[name = tensor("op_836_begin_0"), val = tensor([0, 0, 0])]; + tensor var_836_end_0 = const()[name = tensor("op_836_end_0"), val = tensor([1, 1, 256])]; + tensor var_836_end_mask_0 = const()[name = tensor("op_836_end_mask_0"), val = tensor([true, false, true])]; + tensor var_836 = slice_by_index(begin = var_836_begin_0, end = var_836_end_0, end_mask = var_836_end_mask_0, x = x_15)[name = tensor("op_836")]; + tensor var_839_begin_0 = const()[name = tensor("op_839_begin_0"), val = tensor([0, 1, 0])]; + tensor var_839_end_0 = const()[name = tensor("op_839_end_0"), val = tensor([1, 16, 256])]; + tensor var_839_end_mask_0 = const()[name = tensor("op_839_end_mask_0"), val = tensor([true, true, true])]; + tensor var_839 = slice_by_index(begin = var_839_begin_0, end = var_839_end_0, end_mask = var_839_end_mask_0, x = window_21)[name = tensor("op_839")]; tensor rolled_17_interleave_0 = const()[name = tensor("rolled_17_interleave_0"), val = tensor(false)]; - tensor rolled_17 = concat(axis = var_33, interleave = rolled_17_interleave_0, values = (var_1161, var_1158))[name = tensor("rolled_17")]; + tensor rolled_17 = concat(axis = var_33, interleave = rolled_17_interleave_0, values = (var_839, var_836))[name = tensor("rolled_17")]; tensor clip_16 = clip(alpha = var_21, beta = var_20, x = filled_17)[name = tensor("clip_16")]; - tensor var_1166 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_1166")]; - tensor var_1167_dtype_0 = const()[name = tensor("op_1167_dtype_0"), val = tensor("fp32")]; - tensor var_1168 = const()[name = tensor("op_1168"), val = tensor([1, 16, 1])]; - tensor var_1167 = cast(dtype = var_1167_dtype_0, x = var_1166)[name = tensor("cast_159")]; - tensor fill_mask_17 = reshape(shape = var_1168, x = var_1167)[name = tensor("fill_mask_17")]; + tensor var_844 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_844")]; + tensor var_845_dtype_0 = const()[name = tensor("op_845_dtype_0"), val = tensor("fp32")]; + tensor var_846 = const()[name = tensor("op_846"), val = tensor([1, 16, 1])]; + tensor var_845 = cast(dtype = var_845_dtype_0, x = var_844)[name = tensor("cast_159")]; + tensor fill_mask_17 = reshape(shape = var_846, x = var_845)[name = tensor("fill_mask_17")]; tensor x_expanded_17_reps_0 = const()[name = tensor("x_expanded_17_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_1158)[name = tensor("x_expanded_17")]; - tensor var_1172 = sub(x = var_17, y = fill_mask_17)[name = tensor("op_1172")]; - tensor var_1173 = mul(x = window_5, y = var_1172)[name = tensor("op_1173")]; - tensor var_1174 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_1174")]; - tensor filled_window_17 = add(x = var_1173, y = var_1174)[name = tensor("filled_window_17")]; - tensor var_1176 = greater_equal(x = filled_17, y = var_16)[name = tensor("op_1176")]; - tensor var_1177_dtype_0 = const()[name = tensor("op_1177_dtype_0"), val = tensor("fp32")]; - tensor var_1178 = const()[name = tensor("op_1178"), val = tensor([1, 1, 1])]; - tensor var_1177 = cast(dtype = var_1177_dtype_0, x = var_1176)[name = tensor("cast_158")]; - tensor use_roll_17 = reshape(shape = var_1178, x = var_1177)[name = tensor("use_roll_17")]; - tensor var_1180 = sub(x = var_17, y = use_roll_17)[name = tensor("op_1180")]; - tensor var_1181 = mul(x = filled_window_17, y = var_1180)[name = tensor("op_1181")]; - tensor var_1182 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_1182")]; - tensor input_161 = add(x = var_1181, y = var_1182)[name = tensor("input_161")]; - tensor var_1184 = const()[name = tensor("op_1184"), val = tensor(0x1p+0)]; - tensor var_1185 = add(x = filled_17, y = var_1184)[name = tensor("op_1185")]; - tensor clip_17 = clip(alpha = var_21, beta = var_16, x = var_1185)[name = tensor("clip_17")]; - tensor x_29_axes_0 = const()[name = tensor("x_29_axes_0"), val = tensor([-1])]; - tensor x_29 = layer_norm(axes = x_29_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_161)[name = tensor("x_29")]; - tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_59_pad_type_0 = const()[name = tensor("inputs_59_pad_type_0"), val = tensor("valid")]; - tensor inputs_59_strides_0 = const()[name = tensor("inputs_59_strides_0"), val = tensor([1])]; - tensor inputs_59_pad_0 = const()[name = tensor("inputs_59_pad_0"), val = tensor([0, 0])]; - tensor inputs_59_dilations_0 = const()[name = tensor("inputs_59_dilations_0"), val = tensor([1])]; - tensor inputs_59_groups_0 = const()[name = tensor("inputs_59_groups_0"), val = tensor(1)]; - tensor input_163 = transpose(perm = input_163_perm_0, x = x_29)[name = tensor("transpose_50")]; - tensor inputs_59 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_59_dilations_0, groups = inputs_59_groups_0, pad = inputs_59_pad_0, pad_type = inputs_59_pad_type_0, strides = inputs_59_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_163)[name = tensor("inputs_59")]; - tensor var_1206_split_sizes_0 = const()[name = tensor("op_1206_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1206_axis_0 = const()[name = tensor("op_1206_axis_0"), val = tensor(1)]; - tensor var_1206_0, tensor var_1206_1 = split(axis = var_1206_axis_0, split_sizes = var_1206_split_sizes_0, x = inputs_59)[name = tensor("op_1206")]; - tensor var_1208 = sigmoid(x = var_1206_1)[name = tensor("op_1208")]; - tensor inputs_61 = mul(x = var_1206_0, y = var_1208)[name = tensor("inputs_61")]; - tensor outputs_aug_17_pad_type_0 = const()[name = tensor("outputs_aug_17_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_17_pad_0 = const()[name = tensor("outputs_aug_17_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_17_groups_0 = const()[name = tensor("outputs_aug_17_groups_0"), val = tensor(256)]; - tensor outputs_aug_17_strides_0 = const()[name = tensor("outputs_aug_17_strides_0"), val = tensor([1])]; - tensor outputs_aug_17_dilations_0 = const()[name = tensor("outputs_aug_17_dilations_0"), val = tensor([1])]; - tensor outputs_aug_17 = conv(dilations = outputs_aug_17_dilations_0, groups = outputs_aug_17_groups_0, pad = outputs_aug_17_pad_0, pad_type = outputs_aug_17_pad_type_0, strides = outputs_aug_17_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_61)[name = tensor("outputs_aug_17")]; - tensor input_165_begin_0 = const()[name = tensor("input_165_begin_0"), val = tensor([0, 0, 0])]; - tensor input_165_end_0 = const()[name = tensor("input_165_end_0"), val = tensor([1, 256, 16])]; - tensor input_165_end_mask_0 = const()[name = tensor("input_165_end_mask_0"), val = tensor([true, true, false])]; - tensor input_165 = slice_by_index(begin = input_165_begin_0, end = input_165_end_0, end_mask = input_165_end_mask_0, x = outputs_aug_17)[name = tensor("input_165")]; - tensor inputs_63 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_165)[name = tensor("inputs_63")]; - tensor input_167 = silu(x = inputs_63)[name = tensor("input_167")]; - tensor input_169_pad_type_0 = const()[name = tensor("input_169_pad_type_0"), val = tensor("valid")]; - tensor input_169_strides_0 = const()[name = tensor("input_169_strides_0"), val = tensor([1])]; - tensor input_169_pad_0 = const()[name = tensor("input_169_pad_0"), val = tensor([0, 0])]; - tensor input_169_dilations_0 = const()[name = tensor("input_169_dilations_0"), val = tensor([1])]; - tensor input_169_groups_0 = const()[name = tensor("input_169_groups_0"), val = tensor(1)]; - tensor input_169 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_169_dilations_0, groups = input_169_groups_0, pad = input_169_pad_0, pad_type = input_169_pad_type_0, strides = input_169_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_167)[name = tensor("input_169")]; - tensor conv_out_17_perm_0 = const()[name = tensor("conv_out_17_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1239_begin_0 = const()[name = tensor("op_1239_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1239_end_0 = const()[name = tensor("op_1239_end_0"), val = tensor([1, 16, 256])]; - tensor var_1239_end_mask_0 = const()[name = tensor("op_1239_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_17 = transpose(perm = conv_out_17_perm_0, x = input_169)[name = tensor("transpose_49")]; - tensor var_1239 = slice_by_index(begin = var_1239_begin_0, end = var_1239_end_0, end_mask = var_1239_end_mask_0, x = conv_out_17)[name = tensor("op_1239")]; - tensor var_1242_begin_0 = const()[name = tensor("op_1242_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1242_end_0 = const()[name = tensor("op_1242_end_0"), val = tensor([1, 2, 256])]; - tensor var_1242_end_mask_0 = const()[name = tensor("op_1242_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1242 = slice_by_index(begin = var_1242_begin_0, end = var_1242_end_0, end_mask = var_1242_end_mask_0, x = x_27)[name = tensor("op_1242")]; - tensor var_1245_begin_0 = const()[name = tensor("op_1245_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1245_end_0 = const()[name = tensor("op_1245_end_0"), val = tensor([1, 16, 256])]; - tensor var_1245_end_mask_0 = const()[name = tensor("op_1245_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1245 = slice_by_index(begin = var_1245_begin_0, end = var_1245_end_0, end_mask = var_1245_end_mask_0, x = input_161)[name = tensor("op_1245")]; + tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_836)[name = tensor("x_expanded_17")]; + tensor var_850 = sub(x = var_17, y = fill_mask_17)[name = tensor("op_850")]; + tensor var_851 = mul(x = window_21, y = var_850)[name = tensor("op_851")]; + tensor var_852 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_852")]; + tensor filled_window_17 = add(x = var_851, y = var_852)[name = tensor("filled_window_17")]; + tensor var_854 = greater_equal(x = filled_17, y = var_16)[name = tensor("op_854")]; + tensor var_855_dtype_0 = const()[name = tensor("op_855_dtype_0"), val = tensor("fp32")]; + tensor var_856 = const()[name = tensor("op_856"), val = tensor([1, 1, 1])]; + tensor var_855 = cast(dtype = var_855_dtype_0, x = var_854)[name = tensor("cast_158")]; + tensor use_roll_17 = reshape(shape = var_856, x = var_855)[name = tensor("use_roll_17")]; + tensor var_858 = sub(x = var_17, y = use_roll_17)[name = tensor("op_858")]; + tensor var_859 = mul(x = filled_window_17, y = var_858)[name = tensor("op_859")]; + tensor var_860 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_860")]; + tensor window_23 = add(x = var_859, y = var_860)[name = tensor("window_23")]; + tensor var_862 = const()[name = tensor("op_862"), val = tensor(0x1p+0)]; + tensor var_863 = add(x = filled_17, y = var_862)[name = tensor("op_863")]; + tensor clip_17 = clip(alpha = var_21, beta = var_16, x = var_863)[name = tensor("clip_17")]; + tensor var_866_begin_0 = const()[name = tensor("op_866_begin_0"), val = tensor([0, 1, 0])]; + tensor var_866_end_0 = const()[name = tensor("op_866_end_0"), val = tensor([1, 2, 256])]; + tensor var_866_end_mask_0 = const()[name = tensor("op_866_end_mask_0"), val = tensor([true, false, true])]; + tensor var_866 = slice_by_index(begin = var_866_begin_0, end = var_866_end_0, end_mask = var_866_end_mask_0, x = x_15)[name = tensor("op_866")]; + tensor var_869_begin_0 = const()[name = tensor("op_869_begin_0"), val = tensor([0, 1, 0])]; + tensor var_869_end_0 = const()[name = tensor("op_869_end_0"), val = tensor([1, 16, 256])]; + tensor var_869_end_mask_0 = const()[name = tensor("op_869_end_mask_0"), val = tensor([true, true, true])]; + tensor var_869 = slice_by_index(begin = var_869_begin_0, end = var_869_end_0, end_mask = var_869_end_mask_0, x = window_23)[name = tensor("op_869")]; tensor rolled_19_interleave_0 = const()[name = tensor("rolled_19_interleave_0"), val = tensor(false)]; - tensor rolled_19 = concat(axis = var_33, interleave = rolled_19_interleave_0, values = (var_1245, var_1242))[name = tensor("rolled_19")]; + tensor rolled_19 = concat(axis = var_33, interleave = rolled_19_interleave_0, values = (var_869, var_866))[name = tensor("rolled_19")]; tensor clip_18 = clip(alpha = var_21, beta = var_20, x = clip_17)[name = tensor("clip_18")]; - tensor var_1250 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_1250")]; - tensor var_1251_dtype_0 = const()[name = tensor("op_1251_dtype_0"), val = tensor("fp32")]; - tensor var_1252 = const()[name = tensor("op_1252"), val = tensor([1, 16, 1])]; - tensor var_1251 = cast(dtype = var_1251_dtype_0, x = var_1250)[name = tensor("cast_157")]; - tensor fill_mask_19 = reshape(shape = var_1252, x = var_1251)[name = tensor("fill_mask_19")]; + tensor var_874 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_874")]; + tensor var_875_dtype_0 = const()[name = tensor("op_875_dtype_0"), val = tensor("fp32")]; + tensor var_876 = const()[name = tensor("op_876"), val = tensor([1, 16, 1])]; + tensor var_875 = cast(dtype = var_875_dtype_0, x = var_874)[name = tensor("cast_157")]; + tensor fill_mask_19 = reshape(shape = var_876, x = var_875)[name = tensor("fill_mask_19")]; tensor x_expanded_19_reps_0 = const()[name = tensor("x_expanded_19_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_1242)[name = tensor("x_expanded_19")]; - tensor var_1256 = sub(x = var_17, y = fill_mask_19)[name = tensor("op_1256")]; - tensor var_1257 = mul(x = input_161, y = var_1256)[name = tensor("op_1257")]; - tensor var_1258 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_1258")]; - tensor filled_window_19 = add(x = var_1257, y = var_1258)[name = tensor("filled_window_19")]; - tensor var_1260 = greater_equal(x = clip_17, y = var_16)[name = tensor("op_1260")]; - tensor var_1261_dtype_0 = const()[name = tensor("op_1261_dtype_0"), val = tensor("fp32")]; - tensor var_1262 = const()[name = tensor("op_1262"), val = tensor([1, 1, 1])]; - tensor var_1261 = cast(dtype = var_1261_dtype_0, x = var_1260)[name = tensor("cast_156")]; - tensor use_roll_19 = reshape(shape = var_1262, x = var_1261)[name = tensor("use_roll_19")]; - tensor var_1264 = sub(x = var_17, y = use_roll_19)[name = tensor("op_1264")]; - tensor var_1265 = mul(x = filled_window_19, y = var_1264)[name = tensor("op_1265")]; - tensor var_1266 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_1266")]; - tensor input_171 = add(x = var_1265, y = var_1266)[name = tensor("input_171")]; - tensor var_1268 = const()[name = tensor("op_1268"), val = tensor(0x1p+0)]; - tensor var_1269 = add(x = clip_17, y = var_1268)[name = tensor("op_1269")]; - tensor clip_19 = clip(alpha = var_21, beta = var_16, x = var_1269)[name = tensor("clip_19")]; - tensor x_31_axes_0 = const()[name = tensor("x_31_axes_0"), val = tensor([-1])]; - tensor x_31 = layer_norm(axes = x_31_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_171)[name = tensor("x_31")]; - tensor input_173_perm_0 = const()[name = tensor("input_173_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_65_pad_type_0 = const()[name = tensor("inputs_65_pad_type_0"), val = tensor("valid")]; - tensor inputs_65_strides_0 = const()[name = tensor("inputs_65_strides_0"), val = tensor([1])]; - tensor inputs_65_pad_0 = const()[name = tensor("inputs_65_pad_0"), val = tensor([0, 0])]; - tensor inputs_65_dilations_0 = const()[name = tensor("inputs_65_dilations_0"), val = tensor([1])]; - tensor inputs_65_groups_0 = const()[name = tensor("inputs_65_groups_0"), val = tensor(1)]; - tensor input_173 = transpose(perm = input_173_perm_0, x = x_31)[name = tensor("transpose_48")]; - tensor inputs_65 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_65_dilations_0, groups = inputs_65_groups_0, pad = inputs_65_pad_0, pad_type = inputs_65_pad_type_0, strides = inputs_65_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_173)[name = tensor("inputs_65")]; - tensor var_1290_split_sizes_0 = const()[name = tensor("op_1290_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1290_axis_0 = const()[name = tensor("op_1290_axis_0"), val = tensor(1)]; - tensor var_1290_0, tensor var_1290_1 = split(axis = var_1290_axis_0, split_sizes = var_1290_split_sizes_0, x = inputs_65)[name = tensor("op_1290")]; - tensor var_1292 = sigmoid(x = var_1290_1)[name = tensor("op_1292")]; - tensor inputs_67 = mul(x = var_1290_0, y = var_1292)[name = tensor("inputs_67")]; - tensor outputs_aug_19_pad_type_0 = const()[name = tensor("outputs_aug_19_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_19_pad_0 = const()[name = tensor("outputs_aug_19_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_19_groups_0 = const()[name = tensor("outputs_aug_19_groups_0"), val = tensor(256)]; - tensor outputs_aug_19_strides_0 = const()[name = tensor("outputs_aug_19_strides_0"), val = tensor([1])]; - tensor outputs_aug_19_dilations_0 = const()[name = tensor("outputs_aug_19_dilations_0"), val = tensor([1])]; - tensor outputs_aug_19 = conv(dilations = outputs_aug_19_dilations_0, groups = outputs_aug_19_groups_0, pad = outputs_aug_19_pad_0, pad_type = outputs_aug_19_pad_type_0, strides = outputs_aug_19_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_67)[name = tensor("outputs_aug_19")]; - tensor input_175_begin_0 = const()[name = tensor("input_175_begin_0"), val = tensor([0, 0, 0])]; - tensor input_175_end_0 = const()[name = tensor("input_175_end_0"), val = tensor([1, 256, 16])]; - tensor input_175_end_mask_0 = const()[name = tensor("input_175_end_mask_0"), val = tensor([true, true, false])]; - tensor input_175 = slice_by_index(begin = input_175_begin_0, end = input_175_end_0, end_mask = input_175_end_mask_0, x = outputs_aug_19)[name = tensor("input_175")]; - tensor inputs_69 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_175)[name = tensor("inputs_69")]; - tensor input_177 = silu(x = inputs_69)[name = tensor("input_177")]; - tensor input_179_pad_type_0 = const()[name = tensor("input_179_pad_type_0"), val = tensor("valid")]; - tensor input_179_strides_0 = const()[name = tensor("input_179_strides_0"), val = tensor([1])]; - tensor input_179_pad_0 = const()[name = tensor("input_179_pad_0"), val = tensor([0, 0])]; - tensor input_179_dilations_0 = const()[name = tensor("input_179_dilations_0"), val = tensor([1])]; - tensor input_179_groups_0 = const()[name = tensor("input_179_groups_0"), val = tensor(1)]; - tensor input_179 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_179_dilations_0, groups = input_179_groups_0, pad = input_179_pad_0, pad_type = input_179_pad_type_0, strides = input_179_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_177)[name = tensor("input_179")]; - tensor conv_out_19_perm_0 = const()[name = tensor("conv_out_19_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1323_begin_0 = const()[name = tensor("op_1323_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1323_end_0 = const()[name = tensor("op_1323_end_0"), val = tensor([1, 16, 256])]; - tensor var_1323_end_mask_0 = const()[name = tensor("op_1323_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_19 = transpose(perm = conv_out_19_perm_0, x = input_179)[name = tensor("transpose_47")]; - tensor var_1323 = slice_by_index(begin = var_1323_begin_0, end = var_1323_end_0, end_mask = var_1323_end_mask_0, x = conv_out_19)[name = tensor("op_1323")]; - tensor var_1326_begin_0 = const()[name = tensor("op_1326_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1326_end_0 = const()[name = tensor("op_1326_end_0"), val = tensor([1, 3, 256])]; - tensor var_1326_end_mask_0 = const()[name = tensor("op_1326_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1326 = slice_by_index(begin = var_1326_begin_0, end = var_1326_end_0, end_mask = var_1326_end_mask_0, x = x_27)[name = tensor("op_1326")]; - tensor var_1329_begin_0 = const()[name = tensor("op_1329_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1329_end_0 = const()[name = tensor("op_1329_end_0"), val = tensor([1, 16, 256])]; - tensor var_1329_end_mask_0 = const()[name = tensor("op_1329_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1329 = slice_by_index(begin = var_1329_begin_0, end = var_1329_end_0, end_mask = var_1329_end_mask_0, x = input_171)[name = tensor("op_1329")]; + tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_866)[name = tensor("x_expanded_19")]; + tensor var_880 = sub(x = var_17, y = fill_mask_19)[name = tensor("op_880")]; + tensor var_881 = mul(x = window_23, y = var_880)[name = tensor("op_881")]; + tensor var_882 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_882")]; + tensor filled_window_19 = add(x = var_881, y = var_882)[name = tensor("filled_window_19")]; + tensor var_884 = greater_equal(x = clip_17, y = var_16)[name = tensor("op_884")]; + tensor var_885_dtype_0 = const()[name = tensor("op_885_dtype_0"), val = tensor("fp32")]; + tensor var_886 = const()[name = tensor("op_886"), val = tensor([1, 1, 1])]; + tensor var_885 = cast(dtype = var_885_dtype_0, x = var_884)[name = tensor("cast_156")]; + tensor use_roll_19 = reshape(shape = var_886, x = var_885)[name = tensor("use_roll_19")]; + tensor var_888 = sub(x = var_17, y = use_roll_19)[name = tensor("op_888")]; + tensor var_889 = mul(x = filled_window_19, y = var_888)[name = tensor("op_889")]; + tensor var_890 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_890")]; + tensor window_25 = add(x = var_889, y = var_890)[name = tensor("window_25")]; + tensor var_892 = const()[name = tensor("op_892"), val = tensor(0x1p+0)]; + tensor var_893 = add(x = clip_17, y = var_892)[name = tensor("op_893")]; + tensor clip_19 = clip(alpha = var_21, beta = var_16, x = var_893)[name = tensor("clip_19")]; + tensor var_896_begin_0 = const()[name = tensor("op_896_begin_0"), val = tensor([0, 2, 0])]; + tensor var_896_end_0 = const()[name = tensor("op_896_end_0"), val = tensor([1, 3, 256])]; + tensor var_896_end_mask_0 = const()[name = tensor("op_896_end_mask_0"), val = tensor([true, false, true])]; + tensor var_896 = slice_by_index(begin = var_896_begin_0, end = var_896_end_0, end_mask = var_896_end_mask_0, x = x_15)[name = tensor("op_896")]; + tensor var_899_begin_0 = const()[name = tensor("op_899_begin_0"), val = tensor([0, 1, 0])]; + tensor var_899_end_0 = const()[name = tensor("op_899_end_0"), val = tensor([1, 16, 256])]; + tensor var_899_end_mask_0 = const()[name = tensor("op_899_end_mask_0"), val = tensor([true, true, true])]; + tensor var_899 = slice_by_index(begin = var_899_begin_0, end = var_899_end_0, end_mask = var_899_end_mask_0, x = window_25)[name = tensor("op_899")]; tensor rolled_21_interleave_0 = const()[name = tensor("rolled_21_interleave_0"), val = tensor(false)]; - tensor rolled_21 = concat(axis = var_33, interleave = rolled_21_interleave_0, values = (var_1329, var_1326))[name = tensor("rolled_21")]; + tensor rolled_21 = concat(axis = var_33, interleave = rolled_21_interleave_0, values = (var_899, var_896))[name = tensor("rolled_21")]; tensor clip_20 = clip(alpha = var_21, beta = var_20, x = clip_19)[name = tensor("clip_20")]; - tensor var_1334 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_1334")]; - tensor var_1335_dtype_0 = const()[name = tensor("op_1335_dtype_0"), val = tensor("fp32")]; - tensor var_1336 = const()[name = tensor("op_1336"), val = tensor([1, 16, 1])]; - tensor var_1335 = cast(dtype = var_1335_dtype_0, x = var_1334)[name = tensor("cast_155")]; - tensor fill_mask_21 = reshape(shape = var_1336, x = var_1335)[name = tensor("fill_mask_21")]; + tensor var_904 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_904")]; + tensor var_905_dtype_0 = const()[name = tensor("op_905_dtype_0"), val = tensor("fp32")]; + tensor var_906 = const()[name = tensor("op_906"), val = tensor([1, 16, 1])]; + tensor var_905 = cast(dtype = var_905_dtype_0, x = var_904)[name = tensor("cast_155")]; + tensor fill_mask_21 = reshape(shape = var_906, x = var_905)[name = tensor("fill_mask_21")]; tensor x_expanded_21_reps_0 = const()[name = tensor("x_expanded_21_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_1326)[name = tensor("x_expanded_21")]; - tensor var_1340 = sub(x = var_17, y = fill_mask_21)[name = tensor("op_1340")]; - tensor var_1341 = mul(x = input_171, y = var_1340)[name = tensor("op_1341")]; - tensor var_1342 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_1342")]; - tensor filled_window_21 = add(x = var_1341, y = var_1342)[name = tensor("filled_window_21")]; - tensor var_1344 = greater_equal(x = clip_19, y = var_16)[name = tensor("op_1344")]; - tensor var_1345_dtype_0 = const()[name = tensor("op_1345_dtype_0"), val = tensor("fp32")]; - tensor var_1346 = const()[name = tensor("op_1346"), val = tensor([1, 1, 1])]; - tensor var_1345 = cast(dtype = var_1345_dtype_0, x = var_1344)[name = tensor("cast_154")]; - tensor use_roll_21 = reshape(shape = var_1346, x = var_1345)[name = tensor("use_roll_21")]; - tensor var_1348 = sub(x = var_17, y = use_roll_21)[name = tensor("op_1348")]; - tensor var_1349 = mul(x = filled_window_21, y = var_1348)[name = tensor("op_1349")]; - tensor var_1350 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_1350")]; - tensor input_181 = add(x = var_1349, y = var_1350)[name = tensor("input_181")]; - tensor var_1352 = const()[name = tensor("op_1352"), val = tensor(0x1p+0)]; - tensor var_1353 = add(x = clip_19, y = var_1352)[name = tensor("op_1353")]; - tensor clip_21 = clip(alpha = var_21, beta = var_16, x = var_1353)[name = tensor("clip_21")]; - tensor x_33_axes_0 = const()[name = tensor("x_33_axes_0"), val = tensor([-1])]; - tensor x_33 = layer_norm(axes = x_33_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_181)[name = tensor("x_33")]; - tensor input_183_perm_0 = const()[name = tensor("input_183_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_71_pad_type_0 = const()[name = tensor("inputs_71_pad_type_0"), val = tensor("valid")]; - tensor inputs_71_strides_0 = const()[name = tensor("inputs_71_strides_0"), val = tensor([1])]; - tensor inputs_71_pad_0 = const()[name = tensor("inputs_71_pad_0"), val = tensor([0, 0])]; - tensor inputs_71_dilations_0 = const()[name = tensor("inputs_71_dilations_0"), val = tensor([1])]; - tensor inputs_71_groups_0 = const()[name = tensor("inputs_71_groups_0"), val = tensor(1)]; - tensor input_183 = transpose(perm = input_183_perm_0, x = x_33)[name = tensor("transpose_46")]; - tensor inputs_71 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_71_dilations_0, groups = inputs_71_groups_0, pad = inputs_71_pad_0, pad_type = inputs_71_pad_type_0, strides = inputs_71_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_183)[name = tensor("inputs_71")]; - tensor var_1374_split_sizes_0 = const()[name = tensor("op_1374_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1374_axis_0 = const()[name = tensor("op_1374_axis_0"), val = tensor(1)]; - tensor var_1374_0, tensor var_1374_1 = split(axis = var_1374_axis_0, split_sizes = var_1374_split_sizes_0, x = inputs_71)[name = tensor("op_1374")]; - tensor var_1376 = sigmoid(x = var_1374_1)[name = tensor("op_1376")]; - tensor inputs_73 = mul(x = var_1374_0, y = var_1376)[name = tensor("inputs_73")]; - tensor outputs_aug_21_pad_type_0 = const()[name = tensor("outputs_aug_21_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_21_pad_0 = const()[name = tensor("outputs_aug_21_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_21_groups_0 = const()[name = tensor("outputs_aug_21_groups_0"), val = tensor(256)]; - tensor outputs_aug_21_strides_0 = const()[name = tensor("outputs_aug_21_strides_0"), val = tensor([1])]; - tensor outputs_aug_21_dilations_0 = const()[name = tensor("outputs_aug_21_dilations_0"), val = tensor([1])]; - tensor outputs_aug_21 = conv(dilations = outputs_aug_21_dilations_0, groups = outputs_aug_21_groups_0, pad = outputs_aug_21_pad_0, pad_type = outputs_aug_21_pad_type_0, strides = outputs_aug_21_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_73)[name = tensor("outputs_aug_21")]; - tensor input_185_begin_0 = const()[name = tensor("input_185_begin_0"), val = tensor([0, 0, 0])]; - tensor input_185_end_0 = const()[name = tensor("input_185_end_0"), val = tensor([1, 256, 16])]; - tensor input_185_end_mask_0 = const()[name = tensor("input_185_end_mask_0"), val = tensor([true, true, false])]; - tensor input_185 = slice_by_index(begin = input_185_begin_0, end = input_185_end_0, end_mask = input_185_end_mask_0, x = outputs_aug_21)[name = tensor("input_185")]; - tensor inputs_75 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_185)[name = tensor("inputs_75")]; - tensor input_187 = silu(x = inputs_75)[name = tensor("input_187")]; - tensor input_189_pad_type_0 = const()[name = tensor("input_189_pad_type_0"), val = tensor("valid")]; - tensor input_189_strides_0 = const()[name = tensor("input_189_strides_0"), val = tensor([1])]; - tensor input_189_pad_0 = const()[name = tensor("input_189_pad_0"), val = tensor([0, 0])]; - tensor input_189_dilations_0 = const()[name = tensor("input_189_dilations_0"), val = tensor([1])]; - tensor input_189_groups_0 = const()[name = tensor("input_189_groups_0"), val = tensor(1)]; - tensor input_189 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_189_dilations_0, groups = input_189_groups_0, pad = input_189_pad_0, pad_type = input_189_pad_type_0, strides = input_189_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_187)[name = tensor("input_189")]; - tensor conv_out_21_perm_0 = const()[name = tensor("conv_out_21_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1407_begin_0 = const()[name = tensor("op_1407_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1407_end_0 = const()[name = tensor("op_1407_end_0"), val = tensor([1, 16, 256])]; - tensor var_1407_end_mask_0 = const()[name = tensor("op_1407_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_21 = transpose(perm = conv_out_21_perm_0, x = input_189)[name = tensor("transpose_45")]; - tensor var_1407 = slice_by_index(begin = var_1407_begin_0, end = var_1407_end_0, end_mask = var_1407_end_mask_0, x = conv_out_21)[name = tensor("op_1407")]; - tensor var_1410_begin_0 = const()[name = tensor("op_1410_begin_0"), val = tensor([0, 3, 0])]; - tensor var_1410_end_0 = const()[name = tensor("op_1410_end_0"), val = tensor([1, 1, 256])]; - tensor var_1410_end_mask_0 = const()[name = tensor("op_1410_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1410 = slice_by_index(begin = var_1410_begin_0, end = var_1410_end_0, end_mask = var_1410_end_mask_0, x = x_27)[name = tensor("op_1410")]; - tensor var_1413_begin_0 = const()[name = tensor("op_1413_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1413_end_0 = const()[name = tensor("op_1413_end_0"), val = tensor([1, 16, 256])]; - tensor var_1413_end_mask_0 = const()[name = tensor("op_1413_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1413 = slice_by_index(begin = var_1413_begin_0, end = var_1413_end_0, end_mask = var_1413_end_mask_0, x = input_181)[name = tensor("op_1413")]; + tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_896)[name = tensor("x_expanded_21")]; + tensor var_910 = sub(x = var_17, y = fill_mask_21)[name = tensor("op_910")]; + tensor var_911 = mul(x = window_25, y = var_910)[name = tensor("op_911")]; + tensor var_912 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_912")]; + tensor filled_window_21 = add(x = var_911, y = var_912)[name = tensor("filled_window_21")]; + tensor var_914 = greater_equal(x = clip_19, y = var_16)[name = tensor("op_914")]; + tensor var_915_dtype_0 = const()[name = tensor("op_915_dtype_0"), val = tensor("fp32")]; + tensor var_916 = const()[name = tensor("op_916"), val = tensor([1, 1, 1])]; + tensor var_915 = cast(dtype = var_915_dtype_0, x = var_914)[name = tensor("cast_154")]; + tensor use_roll_21 = reshape(shape = var_916, x = var_915)[name = tensor("use_roll_21")]; + tensor var_918 = sub(x = var_17, y = use_roll_21)[name = tensor("op_918")]; + tensor var_919 = mul(x = filled_window_21, y = var_918)[name = tensor("op_919")]; + tensor var_920 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_920")]; + tensor window_27 = add(x = var_919, y = var_920)[name = tensor("window_27")]; + tensor var_922 = const()[name = tensor("op_922"), val = tensor(0x1p+0)]; + tensor var_923 = add(x = clip_19, y = var_922)[name = tensor("op_923")]; + tensor clip_21 = clip(alpha = var_21, beta = var_16, x = var_923)[name = tensor("clip_21")]; + tensor var_926_begin_0 = const()[name = tensor("op_926_begin_0"), val = tensor([0, 3, 0])]; + tensor var_926_end_0 = const()[name = tensor("op_926_end_0"), val = tensor([1, 1, 256])]; + tensor var_926_end_mask_0 = const()[name = tensor("op_926_end_mask_0"), val = tensor([true, true, true])]; + tensor var_926 = slice_by_index(begin = var_926_begin_0, end = var_926_end_0, end_mask = var_926_end_mask_0, x = x_15)[name = tensor("op_926")]; + tensor var_929_begin_0 = const()[name = tensor("op_929_begin_0"), val = tensor([0, 1, 0])]; + tensor var_929_end_0 = const()[name = tensor("op_929_end_0"), val = tensor([1, 16, 256])]; + tensor var_929_end_mask_0 = const()[name = tensor("op_929_end_mask_0"), val = tensor([true, true, true])]; + tensor var_929 = slice_by_index(begin = var_929_begin_0, end = var_929_end_0, end_mask = var_929_end_mask_0, x = window_27)[name = tensor("op_929")]; tensor rolled_23_interleave_0 = const()[name = tensor("rolled_23_interleave_0"), val = tensor(false)]; - tensor rolled_23 = concat(axis = var_33, interleave = rolled_23_interleave_0, values = (var_1413, var_1410))[name = tensor("rolled_23")]; + tensor rolled_23 = concat(axis = var_33, interleave = rolled_23_interleave_0, values = (var_929, var_926))[name = tensor("rolled_23")]; tensor clip_22 = clip(alpha = var_21, beta = var_20, x = clip_21)[name = tensor("clip_22")]; - tensor var_1418 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_1418")]; - tensor var_1419_dtype_0 = const()[name = tensor("op_1419_dtype_0"), val = tensor("fp32")]; - tensor var_1420 = const()[name = tensor("op_1420"), val = tensor([1, 16, 1])]; - tensor var_1419 = cast(dtype = var_1419_dtype_0, x = var_1418)[name = tensor("cast_153")]; - tensor fill_mask_23 = reshape(shape = var_1420, x = var_1419)[name = tensor("fill_mask_23")]; + tensor var_934 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_934")]; + tensor var_935_dtype_0 = const()[name = tensor("op_935_dtype_0"), val = tensor("fp32")]; + tensor var_936 = const()[name = tensor("op_936"), val = tensor([1, 16, 1])]; + tensor var_935 = cast(dtype = var_935_dtype_0, x = var_934)[name = tensor("cast_153")]; + tensor fill_mask_23 = reshape(shape = var_936, x = var_935)[name = tensor("fill_mask_23")]; tensor x_expanded_23_reps_0 = const()[name = tensor("x_expanded_23_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_23 = tile(reps = x_expanded_23_reps_0, x = var_1410)[name = tensor("x_expanded_23")]; - tensor var_1424 = sub(x = var_17, y = fill_mask_23)[name = tensor("op_1424")]; - tensor var_1425 = mul(x = input_181, y = var_1424)[name = tensor("op_1425")]; - tensor var_1426 = mul(x = x_expanded_23, y = fill_mask_23)[name = tensor("op_1426")]; - tensor filled_window_23 = add(x = var_1425, y = var_1426)[name = tensor("filled_window_23")]; - tensor var_1428 = greater_equal(x = clip_21, y = var_16)[name = tensor("op_1428")]; - tensor var_1429_dtype_0 = const()[name = tensor("op_1429_dtype_0"), val = tensor("fp32")]; - tensor var_1430 = const()[name = tensor("op_1430"), val = tensor([1, 1, 1])]; - tensor var_1429 = cast(dtype = var_1429_dtype_0, x = var_1428)[name = tensor("cast_152")]; - tensor use_roll_23 = reshape(shape = var_1430, x = var_1429)[name = tensor("use_roll_23")]; - tensor var_1432 = sub(x = var_17, y = use_roll_23)[name = tensor("op_1432")]; - tensor var_1433 = mul(x = filled_window_23, y = var_1432)[name = tensor("op_1433")]; - tensor var_1434 = mul(x = rolled_23, y = use_roll_23)[name = tensor("op_1434")]; - tensor input_191 = add(x = var_1433, y = var_1434)[name = tensor("input_191")]; - tensor var_1436 = const()[name = tensor("op_1436"), val = tensor(0x1p+0)]; - tensor var_1437 = add(x = clip_21, y = var_1436)[name = tensor("op_1437")]; - tensor clip_23 = clip(alpha = var_21, beta = var_16, x = var_1437)[name = tensor("clip_23")]; - tensor x_35_axes_0 = const()[name = tensor("x_35_axes_0"), val = tensor([-1])]; - tensor x_35 = layer_norm(axes = x_35_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_191)[name = tensor("x_35")]; - tensor input_193_perm_0 = const()[name = tensor("input_193_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_77_pad_type_0 = const()[name = tensor("inputs_77_pad_type_0"), val = tensor("valid")]; - tensor inputs_77_strides_0 = const()[name = tensor("inputs_77_strides_0"), val = tensor([1])]; - tensor inputs_77_pad_0 = const()[name = tensor("inputs_77_pad_0"), val = tensor([0, 0])]; - tensor inputs_77_dilations_0 = const()[name = tensor("inputs_77_dilations_0"), val = tensor([1])]; - tensor inputs_77_groups_0 = const()[name = tensor("inputs_77_groups_0"), val = tensor(1)]; - tensor input_193 = transpose(perm = input_193_perm_0, x = x_35)[name = tensor("transpose_44")]; - tensor inputs_77 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_77_dilations_0, groups = inputs_77_groups_0, pad = inputs_77_pad_0, pad_type = inputs_77_pad_type_0, strides = inputs_77_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_193)[name = tensor("inputs_77")]; - tensor var_1458_split_sizes_0 = const()[name = tensor("op_1458_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1458_axis_0 = const()[name = tensor("op_1458_axis_0"), val = tensor(1)]; - tensor var_1458_0, tensor var_1458_1 = split(axis = var_1458_axis_0, split_sizes = var_1458_split_sizes_0, x = inputs_77)[name = tensor("op_1458")]; - tensor var_1460 = sigmoid(x = var_1458_1)[name = tensor("op_1460")]; - tensor inputs_79 = mul(x = var_1458_0, y = var_1460)[name = tensor("inputs_79")]; - tensor outputs_aug_23_pad_type_0 = const()[name = tensor("outputs_aug_23_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_23_pad_0 = const()[name = tensor("outputs_aug_23_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_23_groups_0 = const()[name = tensor("outputs_aug_23_groups_0"), val = tensor(256)]; - tensor outputs_aug_23_strides_0 = const()[name = tensor("outputs_aug_23_strides_0"), val = tensor([1])]; - tensor outputs_aug_23_dilations_0 = const()[name = tensor("outputs_aug_23_dilations_0"), val = tensor([1])]; - tensor outputs_aug_23 = conv(dilations = outputs_aug_23_dilations_0, groups = outputs_aug_23_groups_0, pad = outputs_aug_23_pad_0, pad_type = outputs_aug_23_pad_type_0, strides = outputs_aug_23_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_79)[name = tensor("outputs_aug_23")]; - tensor input_195_begin_0 = const()[name = tensor("input_195_begin_0"), val = tensor([0, 0, 0])]; - tensor input_195_end_0 = const()[name = tensor("input_195_end_0"), val = tensor([1, 256, 16])]; - tensor input_195_end_mask_0 = const()[name = tensor("input_195_end_mask_0"), val = tensor([true, true, false])]; - tensor input_195 = slice_by_index(begin = input_195_begin_0, end = input_195_end_0, end_mask = input_195_end_mask_0, x = outputs_aug_23)[name = tensor("input_195")]; - tensor inputs_81 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_195)[name = tensor("inputs_81")]; - tensor input_197 = silu(x = inputs_81)[name = tensor("input_197")]; - tensor input_199_pad_type_0 = const()[name = tensor("input_199_pad_type_0"), val = tensor("valid")]; - tensor input_199_strides_0 = const()[name = tensor("input_199_strides_0"), val = tensor([1])]; - tensor input_199_pad_0 = const()[name = tensor("input_199_pad_0"), val = tensor([0, 0])]; - tensor input_199_dilations_0 = const()[name = tensor("input_199_dilations_0"), val = tensor([1])]; - tensor input_199_groups_0 = const()[name = tensor("input_199_groups_0"), val = tensor(1)]; - tensor input_199 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_199_dilations_0, groups = input_199_groups_0, pad = input_199_pad_0, pad_type = input_199_pad_type_0, strides = input_199_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_197)[name = tensor("input_199")]; - tensor conv_out_23_perm_0 = const()[name = tensor("conv_out_23_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1491_begin_0 = const()[name = tensor("op_1491_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1491_end_0 = const()[name = tensor("op_1491_end_0"), val = tensor([1, 16, 256])]; - tensor var_1491_end_mask_0 = const()[name = tensor("op_1491_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_23 = transpose(perm = conv_out_23_perm_0, x = input_199)[name = tensor("transpose_43")]; - tensor var_1491 = slice_by_index(begin = var_1491_begin_0, end = var_1491_end_0, end_mask = var_1491_end_mask_0, x = conv_out_23)[name = tensor("op_1491")]; - tensor var_1494_interleave_0 = const()[name = tensor("op_1494_interleave_0"), val = tensor(false)]; - tensor var_1494 = concat(axis = var_33, interleave = var_1494_interleave_0, values = (var_1239, var_1323, var_1407, var_1491))[name = tensor("op_1494")]; - tensor input_201 = add(x = x_27, y = var_1494)[name = tensor("input_201")]; - tensor input_203_axes_0 = const()[name = tensor("input_203_axes_0"), val = tensor([-1])]; - tensor input_203 = layer_norm(axes = input_203_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_201)[name = tensor("input_203")]; - tensor inputs_83 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_203)[name = tensor("linear_26")]; - tensor input_205 = silu(x = inputs_83)[name = tensor("input_205")]; - tensor input_209 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_205)[name = tensor("linear_27")]; - tensor var_1517 = const()[name = tensor("op_1517"), val = tensor(0x1p-1)]; - tensor var_1518 = mul(x = input_209, y = var_1517)[name = tensor("op_1518")]; - tensor input_211 = add(x = var_1518, y = input_201)[name = tensor("input_211")]; - tensor input_213_axes_0 = const()[name = tensor("input_213_axes_0"), val = tensor([-1])]; - tensor input_213 = layer_norm(axes = input_213_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_211)[name = tensor("input_213")]; - tensor input_215_axes_0 = const()[name = tensor("input_215_axes_0"), val = tensor([-1])]; - tensor input_215 = layer_norm(axes = input_215_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_213)[name = tensor("input_215")]; - tensor inputs_85 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_215)[name = tensor("linear_28")]; - tensor input_217 = silu(x = inputs_85)[name = tensor("input_217")]; - tensor input_221 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_217)[name = tensor("linear_29")]; - tensor var_1547 = const()[name = tensor("op_1547"), val = tensor(0x1p-1)]; - tensor var_1548 = mul(x = input_221, y = var_1547)[name = tensor("op_1548")]; - tensor input_223 = add(x = var_1548, y = input_213)[name = tensor("input_223")]; - tensor x_37_axes_0 = const()[name = tensor("x_37_axes_0"), val = tensor([-1])]; - tensor x_37 = layer_norm(axes = x_37_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_223)[name = tensor("x_37")]; + tensor x_expanded_23 = tile(reps = x_expanded_23_reps_0, x = var_926)[name = tensor("x_expanded_23")]; + tensor var_940 = sub(x = var_17, y = fill_mask_23)[name = tensor("op_940")]; + tensor var_941 = mul(x = window_27, y = var_940)[name = tensor("op_941")]; + tensor var_942 = mul(x = x_expanded_23, y = fill_mask_23)[name = tensor("op_942")]; + tensor filled_window_23 = add(x = var_941, y = var_942)[name = tensor("filled_window_23")]; + tensor var_944 = greater_equal(x = clip_21, y = var_16)[name = tensor("op_944")]; + tensor var_945_dtype_0 = const()[name = tensor("op_945_dtype_0"), val = tensor("fp32")]; + tensor var_946 = const()[name = tensor("op_946"), val = tensor([1, 1, 1])]; + tensor var_945 = cast(dtype = var_945_dtype_0, x = var_944)[name = tensor("cast_152")]; + tensor use_roll_23 = reshape(shape = var_946, x = var_945)[name = tensor("use_roll_23")]; + tensor var_948 = sub(x = var_17, y = use_roll_23)[name = tensor("op_948")]; + tensor var_949 = mul(x = filled_window_23, y = var_948)[name = tensor("op_949")]; + tensor var_950 = mul(x = rolled_23, y = use_roll_23)[name = tensor("op_950")]; + tensor window_29 = add(x = var_949, y = var_950)[name = tensor("window_29")]; + tensor var_952 = const()[name = tensor("op_952"), val = tensor(0x1p+0)]; + tensor var_953 = add(x = clip_21, y = var_952)[name = tensor("op_953")]; + tensor clip_23 = clip(alpha = var_21, beta = var_16, x = var_953)[name = tensor("clip_23")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_30, interleave = input_101_interleave_0, values = (window_23, window_25, window_27, window_29))[name = tensor("input_101")]; + tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; + tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; + tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_23_pad_type_0 = const()[name = tensor("inputs_23_pad_type_0"), val = tensor("valid")]; + tensor inputs_23_strides_0 = const()[name = tensor("inputs_23_strides_0"), val = tensor([1])]; + tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; + tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; + tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; + tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; + tensor var_976_split_sizes_0 = const()[name = tensor("op_976_split_sizes_0"), val = tensor([256, 256])]; + tensor var_976_axis_0 = const()[name = tensor("op_976_axis_0"), val = tensor(1)]; + tensor var_976_0, tensor var_976_1 = split(axis = var_976_axis_0, split_sizes = var_976_split_sizes_0, x = inputs_23)[name = tensor("op_976")]; + tensor var_978 = sigmoid(x = var_976_1)[name = tensor("op_978")]; + tensor inputs_25 = mul(x = var_976_0, y = var_978)[name = tensor("inputs_25")]; + tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; + tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; + tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; + tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_25)[name = tensor("outputs_aug_5")]; + tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; + tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([4, 256, 16])]; + tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; + tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_5)[name = tensor("input_105")]; + tensor inputs_27 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_105)[name = tensor("inputs_27")]; + tensor input_107 = silu(x = inputs_27)[name = tensor("input_107")]; + tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; + tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; + tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; + tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; + tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; + tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; + tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1009_begin_0 = const()[name = tensor("op_1009_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1009_end_0 = const()[name = tensor("op_1009_end_0"), val = tensor([4, 16, 256])]; + tensor var_1009_end_mask_0 = const()[name = tensor("op_1009_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_1009 = slice_by_index(begin = var_1009_begin_0, end = var_1009_end_0, end_mask = var_1009_end_mask_0, x = conv_out_5)[name = tensor("op_1009")]; + tensor var_1011_perm_0 = const()[name = tensor("op_1011_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1011 = transpose(perm = var_1011_perm_0, x = var_1009)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_1011)[name = tensor("input_111")]; + tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; + tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; + tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; + tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; + tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; + tensor var_1034 = const()[name = tensor("op_1034"), val = tensor(0x1p-1)]; + tensor var_1035 = mul(x = input_119, y = var_1034)[name = tensor("op_1035")]; + tensor input_121 = add(x = var_1035, y = input_111)[name = tensor("input_121")]; + tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; + tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; + tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; + tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; + tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; + tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; + tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; + tensor var_1064 = const()[name = tensor("op_1064"), val = tensor(0x1p-1)]; + tensor var_1065 = mul(x = input_131, y = var_1064)[name = tensor("op_1065")]; + tensor input_133 = add(x = var_1065, y = input_123)[name = tensor("input_133")]; + tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; + tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; tensor prev_kv_7_end_0 = const()[name = tensor("prev_kv_7_end_0"), val = tensor([4, 1, 4, 64, 64])]; tensor prev_kv_7_end_mask_0 = const()[name = tensor("prev_kv_7_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1422,382 +1078,267 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_1562 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_37)[name = tensor("linear_30")]; - tensor var_1563 = const()[name = tensor("op_1563"), val = tensor([1, 4, 4, 64])]; - tensor var_1564 = reshape(shape = var_1563, x = var_1562)[name = tensor("op_1564")]; + tensor var_1079 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_1080 = const()[name = tensor("op_1080"), val = tensor([1, 4, 4, 64])]; + tensor var_1081 = reshape(shape = var_1080, x = var_1079)[name = tensor("op_1081")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1568 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_37)[name = tensor("linear_31")]; - tensor var_1569 = const()[name = tensor("op_1569"), val = tensor(0x1p-3)]; - tensor var_1570 = mul(x = var_1568, y = var_1569)[name = tensor("op_1570")]; - tensor var_1571 = const()[name = tensor("op_1571"), val = tensor([1, 4, 4, 64])]; - tensor var_1572 = reshape(shape = var_1571, x = var_1570)[name = tensor("op_1572")]; + tensor var_1085 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_1086 = const()[name = tensor("op_1086"), val = tensor(0x1p-3)]; + tensor var_1087 = mul(x = var_1085, y = var_1086)[name = tensor("op_1087")]; + tensor var_1088 = const()[name = tensor("op_1088"), val = tensor([1, 4, 4, 64])]; + tensor var_1089 = reshape(shape = var_1088, x = var_1087)[name = tensor("op_1089")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1576 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_37)[name = tensor("linear_32")]; - tensor var_1577 = const()[name = tensor("op_1577"), val = tensor([1, 4, 4, 64])]; - tensor var_1578 = reshape(shape = var_1577, x = var_1576)[name = tensor("op_1578")]; + tensor var_1093 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_1094 = const()[name = tensor("op_1094"), val = tensor([1, 4, 4, 64])]; + tensor var_1095 = reshape(shape = var_1094, x = var_1093)[name = tensor("op_1095")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_227 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_37)[name = tensor("linear_33")]; + tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; tensor s_t_7 = add(x = prev_scale_7, y = encoder__t_index)[name = tensor("s_t_7")]; tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_1572)[name = tensor("transpose_41")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_1564)[name = tensor("transpose_42")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_1089)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_1081)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_1588 = const()[name = tensor("op_1588"), val = tensor([4, 1])]; - tensor var_1589 = reshape(shape = var_1588, x = sqrt_s_t_7)[name = tensor("op_1589")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_1589)[name = tensor("M_7")]; - tensor var_1591 = mul(x = qk_7, y = M_7)[name = tensor("op_1591")]; + tensor var_1105 = const()[name = tensor("op_1105"), val = tensor([4, 1])]; + tensor var_1106 = reshape(shape = var_1105, x = sqrt_s_t_7)[name = tensor("op_1106")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_1106)[name = tensor("M_7")]; + tensor var_1108 = mul(x = qk_7, y = M_7)[name = tensor("op_1108")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_1578)[name = tensor("transpose_40")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1591, y = v_7)[name = tensor("inner_7")]; - tensor var_1593_transpose_x_0 = const()[name = tensor("op_1593_transpose_x_0"), val = tensor(false)]; - tensor var_1593_transpose_y_0 = const()[name = tensor("op_1593_transpose_y_0"), val = tensor(false)]; - tensor var_1593 = matmul(transpose_x = var_1593_transpose_x_0, transpose_y = var_1593_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1593")]; - tensor var_1594 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1594")]; - tensor var_1595 = const()[name = tensor("op_1595"), val = tensor([1, 1, 4, 1])]; - tensor var_1596 = reshape(shape = var_1595, x = var_1594)[name = tensor("op_1596")]; - tensor cross_7 = mul(x = var_1593, y = var_1596)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_1095)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1108, y = v_7)[name = tensor("inner_7")]; + tensor var_1110_transpose_x_0 = const()[name = tensor("op_1110_transpose_x_0"), val = tensor(false)]; + tensor var_1110_transpose_y_0 = const()[name = tensor("op_1110_transpose_y_0"), val = tensor(false)]; + tensor var_1110 = matmul(transpose_x = var_1110_transpose_x_0, transpose_y = var_1110_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1110")]; + tensor var_1111 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1111")]; + tensor var_1112 = const()[name = tensor("op_1112"), val = tensor([1, 1, 4, 1])]; + tensor var_1113 = reshape(shape = var_1112, x = var_1111)[name = tensor("op_1113")]; + tensor cross_7 = mul(x = var_1110, y = var_1113)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_1599 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1599")]; - tensor var_1601_transpose_x_1 = const()[name = tensor("op_1601_transpose_x_1"), val = tensor(true)]; - tensor var_1601_transpose_y_1 = const()[name = tensor("op_1601_transpose_y_1"), val = tensor(false)]; - tensor var_1601 = matmul(transpose_x = var_1601_transpose_x_1, transpose_y = var_1601_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1601")]; - tensor new_kv_unnorm_7 = add(x = var_1599, y = var_1601)[name = tensor("new_kv_unnorm_7")]; - tensor var_1603 = const()[name = tensor("op_1603"), val = tensor(0x1p+2)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_1603)[name = tensor("new_scale_7")]; - tensor var_1605 = sqrt(x = new_scale_7)[name = tensor("op_1605")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1605)[name = tensor("nkv_1")]; - tensor var_1607_perm_0 = const()[name = tensor("op_1607_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1116 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1116")]; + tensor var_1118_transpose_x_1 = const()[name = tensor("op_1118_transpose_x_1"), val = tensor(true)]; + tensor var_1118_transpose_y_1 = const()[name = tensor("op_1118_transpose_y_1"), val = tensor(false)]; + tensor var_1118 = matmul(transpose_x = var_1118_transpose_x_1, transpose_y = var_1118_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1118")]; + tensor new_kv_unnorm_7 = add(x = var_1116, y = var_1118)[name = tensor("new_kv_unnorm_7")]; + tensor var_1120 = const()[name = tensor("op_1120"), val = tensor(0x1p+2)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_1120)[name = tensor("new_scale_7")]; + tensor var_1122 = sqrt(x = new_scale_7)[name = tensor("op_1122")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1122)[name = tensor("nkv_1")]; + tensor var_1124_perm_0 = const()[name = tensor("op_1124_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_1607 = transpose(perm = var_1607_perm_0, x = out_19)[name = tensor("transpose_39")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1607)[name = tensor("out_21")]; - tensor var_1611 = const()[name = tensor("op_1611"), val = tensor([1, 4, 256])]; - tensor out_23 = reshape(shape = var_1611, x = out_21)[name = tensor("out_23")]; - tensor var_1613 = silu(x = input_227)[name = tensor("op_1613")]; - tensor input_229 = mul(x = var_1613, y = out_23)[name = tensor("input_229")]; - tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_229)[name = tensor("linear_34")]; - tensor x_39 = add(x = input_223, y = ret_out_7)[name = tensor("x_39")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor var_1124 = transpose(perm = var_1124_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1124)[name = tensor("out_21")]; + tensor var_1128 = const()[name = tensor("op_1128"), val = tensor([1, 4, 256])]; + tensor out_23 = reshape(shape = var_1128, x = out_21)[name = tensor("out_23")]; + tensor var_1130 = silu(x = input_137)[name = tensor("op_1130")]; + tensor input_139 = mul(x = var_1130, y = out_23)[name = tensor("input_139")]; + tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; + tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; + tensor window_31_begin_0 = const()[name = tensor("window_31_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_31_end_0 = const()[name = tensor("window_31_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_31_end_mask_0 = const()[name = tensor("window_31_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_31_squeeze_mask_0 = const()[name = tensor("window_31_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_31 = slice_by_index(begin = window_31_begin_0, end = window_31_end_0, end_mask = window_31_end_mask_0, squeeze_mask = window_31_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_31")]; tensor filled_25_begin_0 = const()[name = tensor("filled_25_begin_0"), val = tensor([3, 0])]; tensor filled_25_end_0 = const()[name = tensor("filled_25_end_0"), val = tensor([4, 1])]; tensor filled_25_end_mask_0 = const()[name = tensor("filled_25_end_mask_0"), val = tensor([false, true])]; tensor filled_25_squeeze_mask_0 = const()[name = tensor("filled_25_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_25 = slice_by_index(begin = filled_25_begin_0, end = filled_25_end_0, end_mask = filled_25_end_mask_0, squeeze_mask = filled_25_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_25")]; - tensor var_1622_begin_0 = const()[name = tensor("op_1622_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1622_end_0 = const()[name = tensor("op_1622_end_0"), val = tensor([1, 1, 256])]; - tensor var_1622_end_mask_0 = const()[name = tensor("op_1622_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1622 = slice_by_index(begin = var_1622_begin_0, end = var_1622_end_0, end_mask = var_1622_end_mask_0, x = x_39)[name = tensor("op_1622")]; - tensor var_1625_begin_0 = const()[name = tensor("op_1625_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1625_end_0 = const()[name = tensor("op_1625_end_0"), val = tensor([1, 16, 256])]; - tensor var_1625_end_mask_0 = const()[name = tensor("op_1625_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1625 = slice_by_index(begin = var_1625_begin_0, end = var_1625_end_0, end_mask = var_1625_end_mask_0, x = window)[name = tensor("op_1625")]; + tensor var_1139_begin_0 = const()[name = tensor("op_1139_begin_0"), val = tensor([0, 0, 0])]; + tensor var_1139_end_0 = const()[name = tensor("op_1139_end_0"), val = tensor([1, 1, 256])]; + tensor var_1139_end_mask_0 = const()[name = tensor("op_1139_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1139 = slice_by_index(begin = var_1139_begin_0, end = var_1139_end_0, end_mask = var_1139_end_mask_0, x = x_21)[name = tensor("op_1139")]; + tensor var_1142_begin_0 = const()[name = tensor("op_1142_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1142_end_0 = const()[name = tensor("op_1142_end_0"), val = tensor([1, 16, 256])]; + tensor var_1142_end_mask_0 = const()[name = tensor("op_1142_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1142 = slice_by_index(begin = var_1142_begin_0, end = var_1142_end_0, end_mask = var_1142_end_mask_0, x = window_31)[name = tensor("op_1142")]; tensor rolled_25_interleave_0 = const()[name = tensor("rolled_25_interleave_0"), val = tensor(false)]; - tensor rolled_25 = concat(axis = var_33, interleave = rolled_25_interleave_0, values = (var_1625, var_1622))[name = tensor("rolled_25")]; + tensor rolled_25 = concat(axis = var_33, interleave = rolled_25_interleave_0, values = (var_1142, var_1139))[name = tensor("rolled_25")]; tensor clip_24 = clip(alpha = var_21, beta = var_20, x = filled_25)[name = tensor("clip_24")]; - tensor var_1630 = equal(x = encoder__conv_positions, y = clip_24)[name = tensor("op_1630")]; - tensor var_1631_dtype_0 = const()[name = tensor("op_1631_dtype_0"), val = tensor("fp32")]; - tensor var_1632 = const()[name = tensor("op_1632"), val = tensor([1, 16, 1])]; - tensor var_1631 = cast(dtype = var_1631_dtype_0, x = var_1630)[name = tensor("cast_151")]; - tensor fill_mask_25 = reshape(shape = var_1632, x = var_1631)[name = tensor("fill_mask_25")]; + tensor var_1147 = equal(x = encoder__conv_positions, y = clip_24)[name = tensor("op_1147")]; + tensor var_1148_dtype_0 = const()[name = tensor("op_1148_dtype_0"), val = tensor("fp32")]; + tensor var_1149 = const()[name = tensor("op_1149"), val = tensor([1, 16, 1])]; + tensor var_1148 = cast(dtype = var_1148_dtype_0, x = var_1147)[name = tensor("cast_151")]; + tensor fill_mask_25 = reshape(shape = var_1149, x = var_1148)[name = tensor("fill_mask_25")]; tensor x_expanded_25_reps_0 = const()[name = tensor("x_expanded_25_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_25 = tile(reps = x_expanded_25_reps_0, x = var_1622)[name = tensor("x_expanded_25")]; - tensor var_1636 = sub(x = var_17, y = fill_mask_25)[name = tensor("op_1636")]; - tensor var_1637 = mul(x = window, y = var_1636)[name = tensor("op_1637")]; - tensor var_1638 = mul(x = x_expanded_25, y = fill_mask_25)[name = tensor("op_1638")]; - tensor filled_window_25 = add(x = var_1637, y = var_1638)[name = tensor("filled_window_25")]; - tensor var_1640 = greater_equal(x = filled_25, y = var_16)[name = tensor("op_1640")]; - tensor var_1641_dtype_0 = const()[name = tensor("op_1641_dtype_0"), val = tensor("fp32")]; - tensor var_1642 = const()[name = tensor("op_1642"), val = tensor([1, 1, 1])]; - tensor var_1641 = cast(dtype = var_1641_dtype_0, x = var_1640)[name = tensor("cast_150")]; - tensor use_roll_25 = reshape(shape = var_1642, x = var_1641)[name = tensor("use_roll_25")]; - tensor var_1644 = sub(x = var_17, y = use_roll_25)[name = tensor("op_1644")]; - tensor var_1645 = mul(x = filled_window_25, y = var_1644)[name = tensor("op_1645")]; - tensor var_1646 = mul(x = rolled_25, y = use_roll_25)[name = tensor("op_1646")]; - tensor input_231 = add(x = var_1645, y = var_1646)[name = tensor("input_231")]; - tensor var_1648 = const()[name = tensor("op_1648"), val = tensor(0x1p+0)]; - tensor var_1649 = add(x = filled_25, y = var_1648)[name = tensor("op_1649")]; - tensor clip_25 = clip(alpha = var_21, beta = var_16, x = var_1649)[name = tensor("clip_25")]; - tensor x_41_axes_0 = const()[name = tensor("x_41_axes_0"), val = tensor([-1])]; - tensor x_41 = layer_norm(axes = x_41_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_231)[name = tensor("x_41")]; - tensor input_233_perm_0 = const()[name = tensor("input_233_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_87_pad_type_0 = const()[name = tensor("inputs_87_pad_type_0"), val = tensor("valid")]; - tensor inputs_87_strides_0 = const()[name = tensor("inputs_87_strides_0"), val = tensor([1])]; - tensor inputs_87_pad_0 = const()[name = tensor("inputs_87_pad_0"), val = tensor([0, 0])]; - tensor inputs_87_dilations_0 = const()[name = tensor("inputs_87_dilations_0"), val = tensor([1])]; - tensor inputs_87_groups_0 = const()[name = tensor("inputs_87_groups_0"), val = tensor(1)]; - tensor input_233 = transpose(perm = input_233_perm_0, x = x_41)[name = tensor("transpose_38")]; - tensor inputs_87 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_87_dilations_0, groups = inputs_87_groups_0, pad = inputs_87_pad_0, pad_type = inputs_87_pad_type_0, strides = inputs_87_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_233)[name = tensor("inputs_87")]; - tensor var_1670_split_sizes_0 = const()[name = tensor("op_1670_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1670_axis_0 = const()[name = tensor("op_1670_axis_0"), val = tensor(1)]; - tensor var_1670_0, tensor var_1670_1 = split(axis = var_1670_axis_0, split_sizes = var_1670_split_sizes_0, x = inputs_87)[name = tensor("op_1670")]; - tensor var_1672 = sigmoid(x = var_1670_1)[name = tensor("op_1672")]; - tensor inputs_89 = mul(x = var_1670_0, y = var_1672)[name = tensor("inputs_89")]; - tensor outputs_aug_25_pad_type_0 = const()[name = tensor("outputs_aug_25_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_25_pad_0 = const()[name = tensor("outputs_aug_25_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_25_groups_0 = const()[name = tensor("outputs_aug_25_groups_0"), val = tensor(256)]; - tensor outputs_aug_25_strides_0 = const()[name = tensor("outputs_aug_25_strides_0"), val = tensor([1])]; - tensor outputs_aug_25_dilations_0 = const()[name = tensor("outputs_aug_25_dilations_0"), val = tensor([1])]; - tensor outputs_aug_25 = conv(dilations = outputs_aug_25_dilations_0, groups = outputs_aug_25_groups_0, pad = outputs_aug_25_pad_0, pad_type = outputs_aug_25_pad_type_0, strides = outputs_aug_25_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_89)[name = tensor("outputs_aug_25")]; - tensor input_235_begin_0 = const()[name = tensor("input_235_begin_0"), val = tensor([0, 0, 0])]; - tensor input_235_end_0 = const()[name = tensor("input_235_end_0"), val = tensor([1, 256, 16])]; - tensor input_235_end_mask_0 = const()[name = tensor("input_235_end_mask_0"), val = tensor([true, true, false])]; - tensor input_235 = slice_by_index(begin = input_235_begin_0, end = input_235_end_0, end_mask = input_235_end_mask_0, x = outputs_aug_25)[name = tensor("input_235")]; - tensor inputs_91 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_235)[name = tensor("inputs_91")]; - tensor input_237 = silu(x = inputs_91)[name = tensor("input_237")]; - tensor input_239_pad_type_0 = const()[name = tensor("input_239_pad_type_0"), val = tensor("valid")]; - tensor input_239_strides_0 = const()[name = tensor("input_239_strides_0"), val = tensor([1])]; - tensor input_239_pad_0 = const()[name = tensor("input_239_pad_0"), val = tensor([0, 0])]; - tensor input_239_dilations_0 = const()[name = tensor("input_239_dilations_0"), val = tensor([1])]; - tensor input_239_groups_0 = const()[name = tensor("input_239_groups_0"), val = tensor(1)]; - tensor input_239 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_239_dilations_0, groups = input_239_groups_0, pad = input_239_pad_0, pad_type = input_239_pad_type_0, strides = input_239_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_237)[name = tensor("input_239")]; - tensor conv_out_25_perm_0 = const()[name = tensor("conv_out_25_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1703_begin_0 = const()[name = tensor("op_1703_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1703_end_0 = const()[name = tensor("op_1703_end_0"), val = tensor([1, 16, 256])]; - tensor var_1703_end_mask_0 = const()[name = tensor("op_1703_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_25 = transpose(perm = conv_out_25_perm_0, x = input_239)[name = tensor("transpose_37")]; - tensor var_1703 = slice_by_index(begin = var_1703_begin_0, end = var_1703_end_0, end_mask = var_1703_end_mask_0, x = conv_out_25)[name = tensor("op_1703")]; - tensor var_1706_begin_0 = const()[name = tensor("op_1706_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1706_end_0 = const()[name = tensor("op_1706_end_0"), val = tensor([1, 2, 256])]; - tensor var_1706_end_mask_0 = const()[name = tensor("op_1706_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1706 = slice_by_index(begin = var_1706_begin_0, end = var_1706_end_0, end_mask = var_1706_end_mask_0, x = x_39)[name = tensor("op_1706")]; - tensor var_1709_begin_0 = const()[name = tensor("op_1709_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1709_end_0 = const()[name = tensor("op_1709_end_0"), val = tensor([1, 16, 256])]; - tensor var_1709_end_mask_0 = const()[name = tensor("op_1709_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1709 = slice_by_index(begin = var_1709_begin_0, end = var_1709_end_0, end_mask = var_1709_end_mask_0, x = input_231)[name = tensor("op_1709")]; + tensor x_expanded_25 = tile(reps = x_expanded_25_reps_0, x = var_1139)[name = tensor("x_expanded_25")]; + tensor var_1153 = sub(x = var_17, y = fill_mask_25)[name = tensor("op_1153")]; + tensor var_1154 = mul(x = window_31, y = var_1153)[name = tensor("op_1154")]; + tensor var_1155 = mul(x = x_expanded_25, y = fill_mask_25)[name = tensor("op_1155")]; + tensor filled_window_25 = add(x = var_1154, y = var_1155)[name = tensor("filled_window_25")]; + tensor var_1157 = greater_equal(x = filled_25, y = var_16)[name = tensor("op_1157")]; + tensor var_1158_dtype_0 = const()[name = tensor("op_1158_dtype_0"), val = tensor("fp32")]; + tensor var_1159 = const()[name = tensor("op_1159"), val = tensor([1, 1, 1])]; + tensor var_1158 = cast(dtype = var_1158_dtype_0, x = var_1157)[name = tensor("cast_150")]; + tensor use_roll_25 = reshape(shape = var_1159, x = var_1158)[name = tensor("use_roll_25")]; + tensor var_1161 = sub(x = var_17, y = use_roll_25)[name = tensor("op_1161")]; + tensor var_1162 = mul(x = filled_window_25, y = var_1161)[name = tensor("op_1162")]; + tensor var_1163 = mul(x = rolled_25, y = use_roll_25)[name = tensor("op_1163")]; + tensor window_33 = add(x = var_1162, y = var_1163)[name = tensor("window_33")]; + tensor var_1165 = const()[name = tensor("op_1165"), val = tensor(0x1p+0)]; + tensor var_1166 = add(x = filled_25, y = var_1165)[name = tensor("op_1166")]; + tensor clip_25 = clip(alpha = var_21, beta = var_16, x = var_1166)[name = tensor("clip_25")]; + tensor var_1169_begin_0 = const()[name = tensor("op_1169_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1169_end_0 = const()[name = tensor("op_1169_end_0"), val = tensor([1, 2, 256])]; + tensor var_1169_end_mask_0 = const()[name = tensor("op_1169_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1169 = slice_by_index(begin = var_1169_begin_0, end = var_1169_end_0, end_mask = var_1169_end_mask_0, x = x_21)[name = tensor("op_1169")]; + tensor var_1172_begin_0 = const()[name = tensor("op_1172_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1172_end_0 = const()[name = tensor("op_1172_end_0"), val = tensor([1, 16, 256])]; + tensor var_1172_end_mask_0 = const()[name = tensor("op_1172_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1172 = slice_by_index(begin = var_1172_begin_0, end = var_1172_end_0, end_mask = var_1172_end_mask_0, x = window_33)[name = tensor("op_1172")]; tensor rolled_27_interleave_0 = const()[name = tensor("rolled_27_interleave_0"), val = tensor(false)]; - tensor rolled_27 = concat(axis = var_33, interleave = rolled_27_interleave_0, values = (var_1709, var_1706))[name = tensor("rolled_27")]; + tensor rolled_27 = concat(axis = var_33, interleave = rolled_27_interleave_0, values = (var_1172, var_1169))[name = tensor("rolled_27")]; tensor clip_26 = clip(alpha = var_21, beta = var_20, x = clip_25)[name = tensor("clip_26")]; - tensor var_1714 = equal(x = encoder__conv_positions, y = clip_26)[name = tensor("op_1714")]; - tensor var_1715_dtype_0 = const()[name = tensor("op_1715_dtype_0"), val = tensor("fp32")]; - tensor var_1716 = const()[name = tensor("op_1716"), val = tensor([1, 16, 1])]; - tensor var_1715 = cast(dtype = var_1715_dtype_0, x = var_1714)[name = tensor("cast_149")]; - tensor fill_mask_27 = reshape(shape = var_1716, x = var_1715)[name = tensor("fill_mask_27")]; + tensor var_1177 = equal(x = encoder__conv_positions, y = clip_26)[name = tensor("op_1177")]; + tensor var_1178_dtype_0 = const()[name = tensor("op_1178_dtype_0"), val = tensor("fp32")]; + tensor var_1179 = const()[name = tensor("op_1179"), val = tensor([1, 16, 1])]; + tensor var_1178 = cast(dtype = var_1178_dtype_0, x = var_1177)[name = tensor("cast_149")]; + tensor fill_mask_27 = reshape(shape = var_1179, x = var_1178)[name = tensor("fill_mask_27")]; tensor x_expanded_27_reps_0 = const()[name = tensor("x_expanded_27_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_27 = tile(reps = x_expanded_27_reps_0, x = var_1706)[name = tensor("x_expanded_27")]; - tensor var_1720 = sub(x = var_17, y = fill_mask_27)[name = tensor("op_1720")]; - tensor var_1721 = mul(x = input_231, y = var_1720)[name = tensor("op_1721")]; - tensor var_1722 = mul(x = x_expanded_27, y = fill_mask_27)[name = tensor("op_1722")]; - tensor filled_window_27 = add(x = var_1721, y = var_1722)[name = tensor("filled_window_27")]; - tensor var_1724 = greater_equal(x = clip_25, y = var_16)[name = tensor("op_1724")]; - tensor var_1725_dtype_0 = const()[name = tensor("op_1725_dtype_0"), val = tensor("fp32")]; - tensor var_1726 = const()[name = tensor("op_1726"), val = tensor([1, 1, 1])]; - tensor var_1725 = cast(dtype = var_1725_dtype_0, x = var_1724)[name = tensor("cast_148")]; - tensor use_roll_27 = reshape(shape = var_1726, x = var_1725)[name = tensor("use_roll_27")]; - tensor var_1728 = sub(x = var_17, y = use_roll_27)[name = tensor("op_1728")]; - tensor var_1729 = mul(x = filled_window_27, y = var_1728)[name = tensor("op_1729")]; - tensor var_1730 = mul(x = rolled_27, y = use_roll_27)[name = tensor("op_1730")]; - tensor input_241 = add(x = var_1729, y = var_1730)[name = tensor("input_241")]; - tensor var_1732 = const()[name = tensor("op_1732"), val = tensor(0x1p+0)]; - tensor var_1733 = add(x = clip_25, y = var_1732)[name = tensor("op_1733")]; - tensor clip_27 = clip(alpha = var_21, beta = var_16, x = var_1733)[name = tensor("clip_27")]; - tensor x_43_axes_0 = const()[name = tensor("x_43_axes_0"), val = tensor([-1])]; - tensor x_43 = layer_norm(axes = x_43_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_241)[name = tensor("x_43")]; - tensor input_243_perm_0 = const()[name = tensor("input_243_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_93_pad_type_0 = const()[name = tensor("inputs_93_pad_type_0"), val = tensor("valid")]; - tensor inputs_93_strides_0 = const()[name = tensor("inputs_93_strides_0"), val = tensor([1])]; - tensor inputs_93_pad_0 = const()[name = tensor("inputs_93_pad_0"), val = tensor([0, 0])]; - tensor inputs_93_dilations_0 = const()[name = tensor("inputs_93_dilations_0"), val = tensor([1])]; - tensor inputs_93_groups_0 = const()[name = tensor("inputs_93_groups_0"), val = tensor(1)]; - tensor input_243 = transpose(perm = input_243_perm_0, x = x_43)[name = tensor("transpose_36")]; - tensor inputs_93 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_93_dilations_0, groups = inputs_93_groups_0, pad = inputs_93_pad_0, pad_type = inputs_93_pad_type_0, strides = inputs_93_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_243)[name = tensor("inputs_93")]; - tensor var_1754_split_sizes_0 = const()[name = tensor("op_1754_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1754_axis_0 = const()[name = tensor("op_1754_axis_0"), val = tensor(1)]; - tensor var_1754_0, tensor var_1754_1 = split(axis = var_1754_axis_0, split_sizes = var_1754_split_sizes_0, x = inputs_93)[name = tensor("op_1754")]; - tensor var_1756 = sigmoid(x = var_1754_1)[name = tensor("op_1756")]; - tensor inputs_95 = mul(x = var_1754_0, y = var_1756)[name = tensor("inputs_95")]; - tensor outputs_aug_27_pad_type_0 = const()[name = tensor("outputs_aug_27_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_27_pad_0 = const()[name = tensor("outputs_aug_27_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_27_groups_0 = const()[name = tensor("outputs_aug_27_groups_0"), val = tensor(256)]; - tensor outputs_aug_27_strides_0 = const()[name = tensor("outputs_aug_27_strides_0"), val = tensor([1])]; - tensor outputs_aug_27_dilations_0 = const()[name = tensor("outputs_aug_27_dilations_0"), val = tensor([1])]; - tensor outputs_aug_27 = conv(dilations = outputs_aug_27_dilations_0, groups = outputs_aug_27_groups_0, pad = outputs_aug_27_pad_0, pad_type = outputs_aug_27_pad_type_0, strides = outputs_aug_27_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_95)[name = tensor("outputs_aug_27")]; - tensor input_245_begin_0 = const()[name = tensor("input_245_begin_0"), val = tensor([0, 0, 0])]; - tensor input_245_end_0 = const()[name = tensor("input_245_end_0"), val = tensor([1, 256, 16])]; - tensor input_245_end_mask_0 = const()[name = tensor("input_245_end_mask_0"), val = tensor([true, true, false])]; - tensor input_245 = slice_by_index(begin = input_245_begin_0, end = input_245_end_0, end_mask = input_245_end_mask_0, x = outputs_aug_27)[name = tensor("input_245")]; - tensor inputs_97 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_245)[name = tensor("inputs_97")]; - tensor input_247 = silu(x = inputs_97)[name = tensor("input_247")]; - tensor input_249_pad_type_0 = const()[name = tensor("input_249_pad_type_0"), val = tensor("valid")]; - tensor input_249_strides_0 = const()[name = tensor("input_249_strides_0"), val = tensor([1])]; - tensor input_249_pad_0 = const()[name = tensor("input_249_pad_0"), val = tensor([0, 0])]; - tensor input_249_dilations_0 = const()[name = tensor("input_249_dilations_0"), val = tensor([1])]; - tensor input_249_groups_0 = const()[name = tensor("input_249_groups_0"), val = tensor(1)]; - tensor input_249 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_249_dilations_0, groups = input_249_groups_0, pad = input_249_pad_0, pad_type = input_249_pad_type_0, strides = input_249_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_247)[name = tensor("input_249")]; - tensor conv_out_27_perm_0 = const()[name = tensor("conv_out_27_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1787_begin_0 = const()[name = tensor("op_1787_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1787_end_0 = const()[name = tensor("op_1787_end_0"), val = tensor([1, 16, 256])]; - tensor var_1787_end_mask_0 = const()[name = tensor("op_1787_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_27 = transpose(perm = conv_out_27_perm_0, x = input_249)[name = tensor("transpose_35")]; - tensor var_1787 = slice_by_index(begin = var_1787_begin_0, end = var_1787_end_0, end_mask = var_1787_end_mask_0, x = conv_out_27)[name = tensor("op_1787")]; - tensor var_1790_begin_0 = const()[name = tensor("op_1790_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1790_end_0 = const()[name = tensor("op_1790_end_0"), val = tensor([1, 3, 256])]; - tensor var_1790_end_mask_0 = const()[name = tensor("op_1790_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1790 = slice_by_index(begin = var_1790_begin_0, end = var_1790_end_0, end_mask = var_1790_end_mask_0, x = x_39)[name = tensor("op_1790")]; - tensor var_1793_begin_0 = const()[name = tensor("op_1793_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1793_end_0 = const()[name = tensor("op_1793_end_0"), val = tensor([1, 16, 256])]; - tensor var_1793_end_mask_0 = const()[name = tensor("op_1793_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1793 = slice_by_index(begin = var_1793_begin_0, end = var_1793_end_0, end_mask = var_1793_end_mask_0, x = input_241)[name = tensor("op_1793")]; + tensor x_expanded_27 = tile(reps = x_expanded_27_reps_0, x = var_1169)[name = tensor("x_expanded_27")]; + tensor var_1183 = sub(x = var_17, y = fill_mask_27)[name = tensor("op_1183")]; + tensor var_1184 = mul(x = window_33, y = var_1183)[name = tensor("op_1184")]; + tensor var_1185 = mul(x = x_expanded_27, y = fill_mask_27)[name = tensor("op_1185")]; + tensor filled_window_27 = add(x = var_1184, y = var_1185)[name = tensor("filled_window_27")]; + tensor var_1187 = greater_equal(x = clip_25, y = var_16)[name = tensor("op_1187")]; + tensor var_1188_dtype_0 = const()[name = tensor("op_1188_dtype_0"), val = tensor("fp32")]; + tensor var_1189 = const()[name = tensor("op_1189"), val = tensor([1, 1, 1])]; + tensor var_1188 = cast(dtype = var_1188_dtype_0, x = var_1187)[name = tensor("cast_148")]; + tensor use_roll_27 = reshape(shape = var_1189, x = var_1188)[name = tensor("use_roll_27")]; + tensor var_1191 = sub(x = var_17, y = use_roll_27)[name = tensor("op_1191")]; + tensor var_1192 = mul(x = filled_window_27, y = var_1191)[name = tensor("op_1192")]; + tensor var_1193 = mul(x = rolled_27, y = use_roll_27)[name = tensor("op_1193")]; + tensor window_35 = add(x = var_1192, y = var_1193)[name = tensor("window_35")]; + tensor var_1195 = const()[name = tensor("op_1195"), val = tensor(0x1p+0)]; + tensor var_1196 = add(x = clip_25, y = var_1195)[name = tensor("op_1196")]; + tensor clip_27 = clip(alpha = var_21, beta = var_16, x = var_1196)[name = tensor("clip_27")]; + tensor var_1199_begin_0 = const()[name = tensor("op_1199_begin_0"), val = tensor([0, 2, 0])]; + tensor var_1199_end_0 = const()[name = tensor("op_1199_end_0"), val = tensor([1, 3, 256])]; + tensor var_1199_end_mask_0 = const()[name = tensor("op_1199_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1199 = slice_by_index(begin = var_1199_begin_0, end = var_1199_end_0, end_mask = var_1199_end_mask_0, x = x_21)[name = tensor("op_1199")]; + tensor var_1202_begin_0 = const()[name = tensor("op_1202_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1202_end_0 = const()[name = tensor("op_1202_end_0"), val = tensor([1, 16, 256])]; + tensor var_1202_end_mask_0 = const()[name = tensor("op_1202_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1202 = slice_by_index(begin = var_1202_begin_0, end = var_1202_end_0, end_mask = var_1202_end_mask_0, x = window_35)[name = tensor("op_1202")]; tensor rolled_29_interleave_0 = const()[name = tensor("rolled_29_interleave_0"), val = tensor(false)]; - tensor rolled_29 = concat(axis = var_33, interleave = rolled_29_interleave_0, values = (var_1793, var_1790))[name = tensor("rolled_29")]; + tensor rolled_29 = concat(axis = var_33, interleave = rolled_29_interleave_0, values = (var_1202, var_1199))[name = tensor("rolled_29")]; tensor clip_28 = clip(alpha = var_21, beta = var_20, x = clip_27)[name = tensor("clip_28")]; - tensor var_1798 = equal(x = encoder__conv_positions, y = clip_28)[name = tensor("op_1798")]; - tensor var_1799_dtype_0 = const()[name = tensor("op_1799_dtype_0"), val = tensor("fp32")]; - tensor var_1800 = const()[name = tensor("op_1800"), val = tensor([1, 16, 1])]; - tensor var_1799 = cast(dtype = var_1799_dtype_0, x = var_1798)[name = tensor("cast_147")]; - tensor fill_mask_29 = reshape(shape = var_1800, x = var_1799)[name = tensor("fill_mask_29")]; + tensor var_1207 = equal(x = encoder__conv_positions, y = clip_28)[name = tensor("op_1207")]; + tensor var_1208_dtype_0 = const()[name = tensor("op_1208_dtype_0"), val = tensor("fp32")]; + tensor var_1209 = const()[name = tensor("op_1209"), val = tensor([1, 16, 1])]; + tensor var_1208 = cast(dtype = var_1208_dtype_0, x = var_1207)[name = tensor("cast_147")]; + tensor fill_mask_29 = reshape(shape = var_1209, x = var_1208)[name = tensor("fill_mask_29")]; tensor x_expanded_29_reps_0 = const()[name = tensor("x_expanded_29_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_29 = tile(reps = x_expanded_29_reps_0, x = var_1790)[name = tensor("x_expanded_29")]; - tensor var_1804 = sub(x = var_17, y = fill_mask_29)[name = tensor("op_1804")]; - tensor var_1805 = mul(x = input_241, y = var_1804)[name = tensor("op_1805")]; - tensor var_1806 = mul(x = x_expanded_29, y = fill_mask_29)[name = tensor("op_1806")]; - tensor filled_window_29 = add(x = var_1805, y = var_1806)[name = tensor("filled_window_29")]; - tensor var_1808 = greater_equal(x = clip_27, y = var_16)[name = tensor("op_1808")]; - tensor var_1809_dtype_0 = const()[name = tensor("op_1809_dtype_0"), val = tensor("fp32")]; - tensor var_1810 = const()[name = tensor("op_1810"), val = tensor([1, 1, 1])]; - tensor var_1809 = cast(dtype = var_1809_dtype_0, x = var_1808)[name = tensor("cast_146")]; - tensor use_roll_29 = reshape(shape = var_1810, x = var_1809)[name = tensor("use_roll_29")]; - tensor var_1812 = sub(x = var_17, y = use_roll_29)[name = tensor("op_1812")]; - tensor var_1813 = mul(x = filled_window_29, y = var_1812)[name = tensor("op_1813")]; - tensor var_1814 = mul(x = rolled_29, y = use_roll_29)[name = tensor("op_1814")]; - tensor input_251 = add(x = var_1813, y = var_1814)[name = tensor("input_251")]; - tensor var_1816 = const()[name = tensor("op_1816"), val = tensor(0x1p+0)]; - tensor var_1817 = add(x = clip_27, y = var_1816)[name = tensor("op_1817")]; - tensor clip_29 = clip(alpha = var_21, beta = var_16, x = var_1817)[name = tensor("clip_29")]; - tensor x_45_axes_0 = const()[name = tensor("x_45_axes_0"), val = tensor([-1])]; - tensor x_45 = layer_norm(axes = x_45_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_251)[name = tensor("x_45")]; - tensor input_253_perm_0 = const()[name = tensor("input_253_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_99_pad_type_0 = const()[name = tensor("inputs_99_pad_type_0"), val = tensor("valid")]; - tensor inputs_99_strides_0 = const()[name = tensor("inputs_99_strides_0"), val = tensor([1])]; - tensor inputs_99_pad_0 = const()[name = tensor("inputs_99_pad_0"), val = tensor([0, 0])]; - tensor inputs_99_dilations_0 = const()[name = tensor("inputs_99_dilations_0"), val = tensor([1])]; - tensor inputs_99_groups_0 = const()[name = tensor("inputs_99_groups_0"), val = tensor(1)]; - tensor input_253 = transpose(perm = input_253_perm_0, x = x_45)[name = tensor("transpose_34")]; - tensor inputs_99 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_99_dilations_0, groups = inputs_99_groups_0, pad = inputs_99_pad_0, pad_type = inputs_99_pad_type_0, strides = inputs_99_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_253)[name = tensor("inputs_99")]; - tensor var_1838_split_sizes_0 = const()[name = tensor("op_1838_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1838_axis_0 = const()[name = tensor("op_1838_axis_0"), val = tensor(1)]; - tensor var_1838_0, tensor var_1838_1 = split(axis = var_1838_axis_0, split_sizes = var_1838_split_sizes_0, x = inputs_99)[name = tensor("op_1838")]; - tensor var_1840 = sigmoid(x = var_1838_1)[name = tensor("op_1840")]; - tensor inputs_101 = mul(x = var_1838_0, y = var_1840)[name = tensor("inputs_101")]; - tensor outputs_aug_29_pad_type_0 = const()[name = tensor("outputs_aug_29_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_29_pad_0 = const()[name = tensor("outputs_aug_29_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_29_groups_0 = const()[name = tensor("outputs_aug_29_groups_0"), val = tensor(256)]; - tensor outputs_aug_29_strides_0 = const()[name = tensor("outputs_aug_29_strides_0"), val = tensor([1])]; - tensor outputs_aug_29_dilations_0 = const()[name = tensor("outputs_aug_29_dilations_0"), val = tensor([1])]; - tensor outputs_aug_29 = conv(dilations = outputs_aug_29_dilations_0, groups = outputs_aug_29_groups_0, pad = outputs_aug_29_pad_0, pad_type = outputs_aug_29_pad_type_0, strides = outputs_aug_29_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_101)[name = tensor("outputs_aug_29")]; - tensor input_255_begin_0 = const()[name = tensor("input_255_begin_0"), val = tensor([0, 0, 0])]; - tensor input_255_end_0 = const()[name = tensor("input_255_end_0"), val = tensor([1, 256, 16])]; - tensor input_255_end_mask_0 = const()[name = tensor("input_255_end_mask_0"), val = tensor([true, true, false])]; - tensor input_255 = slice_by_index(begin = input_255_begin_0, end = input_255_end_0, end_mask = input_255_end_mask_0, x = outputs_aug_29)[name = tensor("input_255")]; - tensor inputs_103 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_255)[name = tensor("inputs_103")]; - tensor input_257 = silu(x = inputs_103)[name = tensor("input_257")]; - tensor input_259_pad_type_0 = const()[name = tensor("input_259_pad_type_0"), val = tensor("valid")]; - tensor input_259_strides_0 = const()[name = tensor("input_259_strides_0"), val = tensor([1])]; - tensor input_259_pad_0 = const()[name = tensor("input_259_pad_0"), val = tensor([0, 0])]; - tensor input_259_dilations_0 = const()[name = tensor("input_259_dilations_0"), val = tensor([1])]; - tensor input_259_groups_0 = const()[name = tensor("input_259_groups_0"), val = tensor(1)]; - tensor input_259 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_259_dilations_0, groups = input_259_groups_0, pad = input_259_pad_0, pad_type = input_259_pad_type_0, strides = input_259_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_257)[name = tensor("input_259")]; - tensor conv_out_29_perm_0 = const()[name = tensor("conv_out_29_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1871_begin_0 = const()[name = tensor("op_1871_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1871_end_0 = const()[name = tensor("op_1871_end_0"), val = tensor([1, 16, 256])]; - tensor var_1871_end_mask_0 = const()[name = tensor("op_1871_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_29 = transpose(perm = conv_out_29_perm_0, x = input_259)[name = tensor("transpose_33")]; - tensor var_1871 = slice_by_index(begin = var_1871_begin_0, end = var_1871_end_0, end_mask = var_1871_end_mask_0, x = conv_out_29)[name = tensor("op_1871")]; - tensor var_1874_begin_0 = const()[name = tensor("op_1874_begin_0"), val = tensor([0, 3, 0])]; - tensor var_1874_end_0 = const()[name = tensor("op_1874_end_0"), val = tensor([1, 1, 256])]; - tensor var_1874_end_mask_0 = const()[name = tensor("op_1874_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1874 = slice_by_index(begin = var_1874_begin_0, end = var_1874_end_0, end_mask = var_1874_end_mask_0, x = x_39)[name = tensor("op_1874")]; - tensor var_1877_begin_0 = const()[name = tensor("op_1877_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1877_end_0 = const()[name = tensor("op_1877_end_0"), val = tensor([1, 16, 256])]; - tensor var_1877_end_mask_0 = const()[name = tensor("op_1877_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1877 = slice_by_index(begin = var_1877_begin_0, end = var_1877_end_0, end_mask = var_1877_end_mask_0, x = input_251)[name = tensor("op_1877")]; + tensor x_expanded_29 = tile(reps = x_expanded_29_reps_0, x = var_1199)[name = tensor("x_expanded_29")]; + tensor var_1213 = sub(x = var_17, y = fill_mask_29)[name = tensor("op_1213")]; + tensor var_1214 = mul(x = window_35, y = var_1213)[name = tensor("op_1214")]; + tensor var_1215 = mul(x = x_expanded_29, y = fill_mask_29)[name = tensor("op_1215")]; + tensor filled_window_29 = add(x = var_1214, y = var_1215)[name = tensor("filled_window_29")]; + tensor var_1217 = greater_equal(x = clip_27, y = var_16)[name = tensor("op_1217")]; + tensor var_1218_dtype_0 = const()[name = tensor("op_1218_dtype_0"), val = tensor("fp32")]; + tensor var_1219 = const()[name = tensor("op_1219"), val = tensor([1, 1, 1])]; + tensor var_1218 = cast(dtype = var_1218_dtype_0, x = var_1217)[name = tensor("cast_146")]; + tensor use_roll_29 = reshape(shape = var_1219, x = var_1218)[name = tensor("use_roll_29")]; + tensor var_1221 = sub(x = var_17, y = use_roll_29)[name = tensor("op_1221")]; + tensor var_1222 = mul(x = filled_window_29, y = var_1221)[name = tensor("op_1222")]; + tensor var_1223 = mul(x = rolled_29, y = use_roll_29)[name = tensor("op_1223")]; + tensor window_37 = add(x = var_1222, y = var_1223)[name = tensor("window_37")]; + tensor var_1225 = const()[name = tensor("op_1225"), val = tensor(0x1p+0)]; + tensor var_1226 = add(x = clip_27, y = var_1225)[name = tensor("op_1226")]; + tensor clip_29 = clip(alpha = var_21, beta = var_16, x = var_1226)[name = tensor("clip_29")]; + tensor var_1229_begin_0 = const()[name = tensor("op_1229_begin_0"), val = tensor([0, 3, 0])]; + tensor var_1229_end_0 = const()[name = tensor("op_1229_end_0"), val = tensor([1, 1, 256])]; + tensor var_1229_end_mask_0 = const()[name = tensor("op_1229_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1229 = slice_by_index(begin = var_1229_begin_0, end = var_1229_end_0, end_mask = var_1229_end_mask_0, x = x_21)[name = tensor("op_1229")]; + tensor var_1232_begin_0 = const()[name = tensor("op_1232_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1232_end_0 = const()[name = tensor("op_1232_end_0"), val = tensor([1, 16, 256])]; + tensor var_1232_end_mask_0 = const()[name = tensor("op_1232_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1232 = slice_by_index(begin = var_1232_begin_0, end = var_1232_end_0, end_mask = var_1232_end_mask_0, x = window_37)[name = tensor("op_1232")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1877, var_1874))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1232, var_1229))[name = tensor("rolled")]; tensor clip_30 = clip(alpha = var_21, beta = var_20, x = clip_29)[name = tensor("clip_30")]; - tensor var_1882 = equal(x = encoder__conv_positions, y = clip_30)[name = tensor("op_1882")]; - tensor var_1883_dtype_0 = const()[name = tensor("op_1883_dtype_0"), val = tensor("fp32")]; - tensor var_1884 = const()[name = tensor("op_1884"), val = tensor([1, 16, 1])]; - tensor var_1883 = cast(dtype = var_1883_dtype_0, x = var_1882)[name = tensor("cast_145")]; - tensor fill_mask = reshape(shape = var_1884, x = var_1883)[name = tensor("fill_mask")]; + tensor var_1237 = equal(x = encoder__conv_positions, y = clip_30)[name = tensor("op_1237")]; + tensor var_1238_dtype_0 = const()[name = tensor("op_1238_dtype_0"), val = tensor("fp32")]; + tensor var_1239 = const()[name = tensor("op_1239"), val = tensor([1, 16, 1])]; + tensor var_1238 = cast(dtype = var_1238_dtype_0, x = var_1237)[name = tensor("cast_145")]; + tensor fill_mask = reshape(shape = var_1239, x = var_1238)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1874)[name = tensor("x_expanded")]; - tensor var_1888 = sub(x = var_17, y = fill_mask)[name = tensor("op_1888")]; - tensor var_1889 = mul(x = input_251, y = var_1888)[name = tensor("op_1889")]; - tensor var_1890 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1890")]; - tensor filled_window = add(x = var_1889, y = var_1890)[name = tensor("filled_window")]; - tensor var_1892 = greater_equal(x = clip_29, y = var_16)[name = tensor("op_1892")]; - tensor var_1893_dtype_0 = const()[name = tensor("op_1893_dtype_0"), val = tensor("fp32")]; - tensor var_1894 = const()[name = tensor("op_1894"), val = tensor([1, 1, 1])]; - tensor var_1893 = cast(dtype = var_1893_dtype_0, x = var_1892)[name = tensor("cast_144")]; - tensor use_roll = reshape(shape = var_1894, x = var_1893)[name = tensor("use_roll")]; - tensor var_1896 = sub(x = var_17, y = use_roll)[name = tensor("op_1896")]; - tensor var_1897 = mul(x = filled_window, y = var_1896)[name = tensor("op_1897")]; - tensor var_1898 = mul(x = rolled, y = use_roll)[name = tensor("op_1898")]; - tensor input_261 = add(x = var_1897, y = var_1898)[name = tensor("input_261")]; - tensor var_1900 = const()[name = tensor("op_1900"), val = tensor(0x1p+0)]; - tensor var_1901 = add(x = clip_29, y = var_1900)[name = tensor("op_1901")]; - tensor clip_31 = clip(alpha = var_21, beta = var_16, x = var_1901)[name = tensor("clip_31")]; - tensor x_47_axes_0 = const()[name = tensor("x_47_axes_0"), val = tensor([-1])]; - tensor x_47 = layer_norm(axes = x_47_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_261)[name = tensor("x_47")]; - tensor input_263_perm_0 = const()[name = tensor("input_263_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_105_pad_type_0 = const()[name = tensor("inputs_105_pad_type_0"), val = tensor("valid")]; - tensor inputs_105_strides_0 = const()[name = tensor("inputs_105_strides_0"), val = tensor([1])]; - tensor inputs_105_pad_0 = const()[name = tensor("inputs_105_pad_0"), val = tensor([0, 0])]; - tensor inputs_105_dilations_0 = const()[name = tensor("inputs_105_dilations_0"), val = tensor([1])]; - tensor inputs_105_groups_0 = const()[name = tensor("inputs_105_groups_0"), val = tensor(1)]; - tensor input_263 = transpose(perm = input_263_perm_0, x = x_47)[name = tensor("transpose_32")]; - tensor inputs_105 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_105_dilations_0, groups = inputs_105_groups_0, pad = inputs_105_pad_0, pad_type = inputs_105_pad_type_0, strides = inputs_105_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_263)[name = tensor("inputs_105")]; - tensor var_1922_split_sizes_0 = const()[name = tensor("op_1922_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1922_axis_0 = const()[name = tensor("op_1922_axis_0"), val = tensor(1)]; - tensor var_1922_0, tensor var_1922_1 = split(axis = var_1922_axis_0, split_sizes = var_1922_split_sizes_0, x = inputs_105)[name = tensor("op_1922")]; - tensor var_1924 = sigmoid(x = var_1922_1)[name = tensor("op_1924")]; - tensor inputs_107 = mul(x = var_1922_0, y = var_1924)[name = tensor("inputs_107")]; + tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1229)[name = tensor("x_expanded")]; + tensor var_1243 = sub(x = var_17, y = fill_mask)[name = tensor("op_1243")]; + tensor var_1244 = mul(x = window_37, y = var_1243)[name = tensor("op_1244")]; + tensor var_1245 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1245")]; + tensor filled_window = add(x = var_1244, y = var_1245)[name = tensor("filled_window")]; + tensor var_1247 = greater_equal(x = clip_29, y = var_16)[name = tensor("op_1247")]; + tensor var_1248_dtype_0 = const()[name = tensor("op_1248_dtype_0"), val = tensor("fp32")]; + tensor var_1249 = const()[name = tensor("op_1249"), val = tensor([1, 1, 1])]; + tensor var_1248 = cast(dtype = var_1248_dtype_0, x = var_1247)[name = tensor("cast_144")]; + tensor use_roll = reshape(shape = var_1249, x = var_1248)[name = tensor("use_roll")]; + tensor var_1251 = sub(x = var_17, y = use_roll)[name = tensor("op_1251")]; + tensor var_1252 = mul(x = filled_window, y = var_1251)[name = tensor("op_1252")]; + tensor var_1253 = mul(x = rolled, y = use_roll)[name = tensor("op_1253")]; + tensor window = add(x = var_1252, y = var_1253)[name = tensor("window")]; + tensor var_1255 = const()[name = tensor("op_1255"), val = tensor(0x1p+0)]; + tensor var_1256 = add(x = clip_29, y = var_1255)[name = tensor("op_1256")]; + tensor clip_31 = clip(alpha = var_21, beta = var_16, x = var_1256)[name = tensor("clip_31")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_30, interleave = input_141_interleave_0, values = (window_33, window_35, window_37, window))[name = tensor("input_141")]; + tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; + tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; + tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_33_pad_type_0 = const()[name = tensor("inputs_33_pad_type_0"), val = tensor("valid")]; + tensor inputs_33_strides_0 = const()[name = tensor("inputs_33_strides_0"), val = tensor([1])]; + tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; + tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; + tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; + tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; + tensor var_1279_split_sizes_0 = const()[name = tensor("op_1279_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1279_axis_0 = const()[name = tensor("op_1279_axis_0"), val = tensor(1)]; + tensor var_1279_0, tensor var_1279_1 = split(axis = var_1279_axis_0, split_sizes = var_1279_split_sizes_0, x = inputs_33)[name = tensor("op_1279")]; + tensor var_1281 = sigmoid(x = var_1279_1)[name = tensor("op_1281")]; + tensor inputs_35 = mul(x = var_1279_0, y = var_1281)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; tensor outputs_aug_strides_0 = const()[name = tensor("outputs_aug_strides_0"), val = tensor([1])]; tensor outputs_aug_dilations_0 = const()[name = tensor("outputs_aug_dilations_0"), val = tensor([1])]; - tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_107)[name = tensor("outputs_aug")]; - tensor input_265_begin_0 = const()[name = tensor("input_265_begin_0"), val = tensor([0, 0, 0])]; - tensor input_265_end_0 = const()[name = tensor("input_265_end_0"), val = tensor([1, 256, 16])]; - tensor input_265_end_mask_0 = const()[name = tensor("input_265_end_mask_0"), val = tensor([true, true, false])]; - tensor input_265 = slice_by_index(begin = input_265_begin_0, end = input_265_end_0, end_mask = input_265_end_mask_0, x = outputs_aug)[name = tensor("input_265")]; - tensor inputs_109 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_265)[name = tensor("inputs_109")]; - tensor input_267 = silu(x = inputs_109)[name = tensor("input_267")]; - tensor input_269_pad_type_0 = const()[name = tensor("input_269_pad_type_0"), val = tensor("valid")]; - tensor input_269_strides_0 = const()[name = tensor("input_269_strides_0"), val = tensor([1])]; - tensor input_269_pad_0 = const()[name = tensor("input_269_pad_0"), val = tensor([0, 0])]; - tensor input_269_dilations_0 = const()[name = tensor("input_269_dilations_0"), val = tensor([1])]; - tensor input_269_groups_0 = const()[name = tensor("input_269_groups_0"), val = tensor(1)]; - tensor input_269 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_269_dilations_0, groups = input_269_groups_0, pad = input_269_pad_0, pad_type = input_269_pad_type_0, strides = input_269_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_267)[name = tensor("input_269")]; - tensor conv_out_31_perm_0 = const()[name = tensor("conv_out_31_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1955_begin_0 = const()[name = tensor("op_1955_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1955_end_0 = const()[name = tensor("op_1955_end_0"), val = tensor([1, 16, 256])]; - tensor var_1955_end_mask_0 = const()[name = tensor("op_1955_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_31 = transpose(perm = conv_out_31_perm_0, x = input_269)[name = tensor("transpose_31")]; - tensor var_1955 = slice_by_index(begin = var_1955_begin_0, end = var_1955_end_0, end_mask = var_1955_end_mask_0, x = conv_out_31)[name = tensor("op_1955")]; - tensor var_1958_interleave_0 = const()[name = tensor("op_1958_interleave_0"), val = tensor(false)]; - tensor var_1958 = concat(axis = var_33, interleave = var_1958_interleave_0, values = (var_1703, var_1787, var_1871, var_1955))[name = tensor("op_1958")]; - tensor input_271 = add(x = x_39, y = var_1958)[name = tensor("input_271")]; - tensor input_273_axes_0 = const()[name = tensor("input_273_axes_0"), val = tensor([-1])]; - tensor input_273 = layer_norm(axes = input_273_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_271)[name = tensor("input_273")]; - tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_273)[name = tensor("linear_35")]; - tensor input_275 = silu(x = inputs)[name = tensor("input_275")]; - tensor input_279 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_275)[name = tensor("linear_36")]; - tensor var_1981 = const()[name = tensor("op_1981"), val = tensor(0x1p-1)]; - tensor var_1982 = mul(x = input_279, y = var_1981)[name = tensor("op_1982")]; - tensor input_281 = add(x = var_1982, y = input_271)[name = tensor("input_281")]; - tensor x_49_axes_0 = const()[name = tensor("x_49_axes_0"), val = tensor([-1])]; - tensor x_49 = layer_norm(axes = x_49_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_281)[name = tensor("x_49")]; + tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_35)[name = tensor("outputs_aug")]; + tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; + tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([4, 256, 16])]; + tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; + tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug)[name = tensor("input_145")]; + tensor inputs_37 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_145)[name = tensor("inputs_37")]; + tensor input_147 = silu(x = inputs_37)[name = tensor("input_147")]; + tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; + tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; + tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; + tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; + tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; + tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; + tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1312_begin_0 = const()[name = tensor("op_1312_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1312_end_0 = const()[name = tensor("op_1312_end_0"), val = tensor([4, 16, 256])]; + tensor var_1312_end_mask_0 = const()[name = tensor("op_1312_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_1312 = slice_by_index(begin = var_1312_begin_0, end = var_1312_end_0, end_mask = var_1312_end_mask_0, x = conv_out_7)[name = tensor("op_1312")]; + tensor var_1314_perm_0 = const()[name = tensor("op_1314_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1314 = transpose(perm = var_1314_perm_0, x = var_1312)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_1314)[name = tensor("input_151")]; + tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; + tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; + tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; + tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; + tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; + tensor var_1337 = const()[name = tensor("op_1337"), val = tensor(0x1p-1)]; + tensor var_1338 = mul(x = input_159, y = var_1337)[name = tensor("op_1338")]; + tensor input_161 = add(x = var_1338, y = input_151)[name = tensor("input_161")]; + tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; + tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; tensor cat_interleave_0 = const()[name = tensor("cat_interleave_0"), val = tensor(false)]; - tensor x_ct = transpose(perm = x_ct_perm_0, x = x_49)[name = tensor("transpose_30")]; + tensor x_ct = transpose(perm = x_ct_perm_0, x = x_25)[name = tensor("transpose_30")]; tensor cat = concat(axis = var_27, interleave = cat_interleave_0, values = (cnn_window, x_ct))[name = tensor("cat")]; tensor conv_out_pad_type_0 = const()[name = tensor("conv_out_pad_type_0"), val = tensor("valid")]; tensor conv_out_strides_0 = const()[name = tensor("conv_out_strides_0"), val = tensor([1])]; @@ -1805,44 +1346,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_2000_begin_0 = const()[name = tensor("op_2000_begin_0"), val = tensor([0, 0, 4])]; - tensor var_2000_end_0 = const()[name = tensor("op_2000_end_0"), val = tensor([1, 256, 22])]; - tensor var_2000_end_mask_0 = const()[name = tensor("op_2000_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_2000_begin_0, end = var_2000_end_0, end_mask = var_2000_end_mask_0, x = cat)[name = tensor("op_2000")]; - tensor input_283_perm_0 = const()[name = tensor("input_283_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2002 = const()[name = tensor("op_2002"), val = tensor([-1])]; - tensor input_283 = transpose(perm = input_283_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_2003 = reduce_l2_norm(axes = var_2002, keep_dims = var_36, x = input_283)[name = tensor("op_2003")]; - tensor const_24 = const()[name = tensor("const_24"), val = tensor(0x1.fffffep+127)]; - tensor clip_32 = clip(alpha = var_12, beta = const_24, x = var_2003)[name = tensor("clip_32")]; - tensor emb = real_div(x = input_283, y = clip_32)[name = tensor("emb")]; - tensor var_2007_axis_0 = const()[name = tensor("op_2007_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_2007_axis_0, values = (var_214, var_678, var_1142, nkv_1))[name = tensor("op_2007")]; - tensor var_2009_axis_0 = const()[name = tensor("op_2009_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_2009_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_2009")]; - tensor var_2011_axis_0 = const()[name = tensor("op_2011_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_2011_axis_0, values = (input_51, input_121, input_191, input_261))[name = tensor("op_2011")]; - tensor var_2013_axis_0 = const()[name = tensor("op_2013_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_2013_axis_0, values = (clip_7, clip_15, clip_23, clip_31))[name = tensor("op_2013")]; - tensor var_2023 = const()[name = tensor("op_2023"), val = tensor(0x1.5798eep-27)]; - tensor var_2029 = const()[name = tensor("op_2029"), val = tensor(0x1.4f8b58p-17)]; - tensor var_2031 = const()[name = tensor("op_2031"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_2032 = const()[name = tensor("op_2032"), val = tensor(true)]; - tensor var_2034 = const()[name = tensor("op_2034"), val = tensor(0x1p+0)]; - tensor var_2038 = const()[name = tensor("op_2038"), val = tensor(-1)]; - tensor var_2043 = const()[name = tensor("op_2043"), val = tensor(0)]; + tensor var_1356_begin_0 = const()[name = tensor("op_1356_begin_0"), val = tensor([0, 0, 4])]; + tensor var_1356_end_0 = const()[name = tensor("op_1356_end_0"), val = tensor([1, 256, 22])]; + tensor var_1356_end_mask_0 = const()[name = tensor("op_1356_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_1356_begin_0, end = var_1356_end_0, end_mask = var_1356_end_mask_0, x = cat)[name = tensor("op_1356")]; + tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1358 = const()[name = tensor("op_1358"), val = tensor([-1])]; + tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; + tensor var_1359 = reduce_l2_norm(axes = var_1358, keep_dims = var_36, x = input_163)[name = tensor("op_1359")]; + tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; + tensor clip_32 = clip(alpha = var_12, beta = const_12, x = var_1359)[name = tensor("clip_32")]; + tensor emb = real_div(x = input_163, y = clip_32)[name = tensor("emb")]; + tensor var_1363_axis_0 = const()[name = tensor("op_1363_axis_0"), val = tensor(0)]; + tensor enc_kv_new = stack(axis = var_1363_axis_0, values = (var_214, var_517, var_820, nkv_1))[name = tensor("op_1363")]; + tensor var_1365_axis_0 = const()[name = tensor("op_1365_axis_0"), val = tensor(0)]; + tensor enc_scale_new = stack(axis = var_1365_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1365")]; + tensor var_1367_axis_0 = const()[name = tensor("op_1367_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1367_axis_0, values = (window_9, window_19, window_29, window))[name = tensor("op_1367")]; + tensor var_1369_axis_0 = const()[name = tensor("op_1369_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1369_axis_0, values = (clip_7, clip_15, clip_23, clip_31))[name = tensor("op_1369")]; + tensor var_1379 = const()[name = tensor("op_1379"), val = tensor(0x1.5798eep-27)]; + tensor var_1384 = const()[name = tensor("op_1384"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1386 = const()[name = tensor("op_1386"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1387 = const()[name = tensor("op_1387"), val = tensor(true)]; + tensor var_1389 = const()[name = tensor("op_1389"), val = tensor(0x1p+0)]; + tensor var_1393 = const()[name = tensor("op_1393"), val = tensor(-1)]; + tensor var_1399 = const()[name = tensor("op_1399"), val = tensor(0)]; tensor pos = const()[name = tensor("pos"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395712)))]; - tensor var_2105_axes_0 = const()[name = tensor("op_2105_axes_0"), val = tensor([2])]; - tensor var_2105 = expand_dims(axes = var_2105_axes_0, x = emb)[name = tensor("op_2105")]; - tensor var_2106 = const()[name = tensor("op_2106"), val = tensor([1, 1, 6, 1])]; - tensor emb_exp = tile(reps = var_2106, x = var_2105)[name = tensor("emb_exp")]; - tensor input_285_interleave_0 = const()[name = tensor("input_285_interleave_0"), val = tensor(false)]; - tensor input_285 = concat(axis = var_2038, interleave = input_285_interleave_0, values = (emb_exp, pos))[name = tensor("input_285")]; - tensor x_51 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_285)[name = tensor("linear_37")]; - tensor var_2113_perm_0 = const()[name = tensor("op_2113_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2117 = const()[name = tensor("op_2117"), val = tensor([6, 4, 256])]; - tensor var_2113 = transpose(perm = var_2113_perm_0, x = x_51)[name = tensor("transpose_28")]; - tensor x_53 = reshape(shape = var_2117, x = var_2113)[name = tensor("x_53")]; + tensor var_1461_axes_0 = const()[name = tensor("op_1461_axes_0"), val = tensor([2])]; + tensor var_1461 = expand_dims(axes = var_1461_axes_0, x = emb)[name = tensor("op_1461")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 6, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1461)[name = tensor("emb_exp")]; + tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; + tensor input_165 = concat(axis = var_1393, interleave = input_165_interleave_0, values = (emb_exp, pos))[name = tensor("input_165")]; + tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; + tensor var_1469_perm_0 = const()[name = tensor("op_1469_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1473 = const()[name = tensor("op_1473"), val = tensor([6, 4, 256])]; + tensor var_1469 = transpose(perm = var_1469_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1473, x = var_1469)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 6, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1853,132 +1394,132 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_2125 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_53)[name = tensor("linear_38")]; - tensor var_2126 = const()[name = tensor("op_2126"), val = tensor([6, 4, 4, 64])]; - tensor var_2127 = reshape(shape = var_2126, x = var_2125)[name = tensor("op_2127")]; + tensor var_1481 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1482 = const()[name = tensor("op_1482"), val = tensor([6, 4, 4, 64])]; + tensor var_1483 = reshape(shape = var_1482, x = var_1481)[name = tensor("op_1483")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2131 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_53)[name = tensor("linear_39")]; - tensor var_2132 = const()[name = tensor("op_2132"), val = tensor(0x1p-3)]; - tensor var_2133 = mul(x = var_2131, y = var_2132)[name = tensor("op_2133")]; - tensor var_2134 = const()[name = tensor("op_2134"), val = tensor([6, 4, 4, 64])]; - tensor var_2135 = reshape(shape = var_2134, x = var_2133)[name = tensor("op_2135")]; + tensor var_1487 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1488 = const()[name = tensor("op_1488"), val = tensor(0x1p-3)]; + tensor var_1489 = mul(x = var_1487, y = var_1488)[name = tensor("op_1489")]; + tensor var_1490 = const()[name = tensor("op_1490"), val = tensor([6, 4, 4, 64])]; + tensor var_1491 = reshape(shape = var_1490, x = var_1489)[name = tensor("op_1491")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2139 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_53)[name = tensor("linear_40")]; - tensor var_2140 = const()[name = tensor("op_2140"), val = tensor([6, 4, 4, 64])]; - tensor var_2141 = reshape(shape = var_2140, x = var_2139)[name = tensor("op_2141")]; + tensor var_1495 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1496 = const()[name = tensor("op_1496"), val = tensor([6, 4, 4, 64])]; + tensor var_1497 = reshape(shape = var_1496, x = var_1495)[name = tensor("op_1497")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_289 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_53)[name = tensor("linear_41")]; + tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_2043, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1399, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; - tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; - tensor clip_33 = clip(alpha = var_2034, beta = const_32, x = s_t_9)[name = tensor("clip_33")]; + tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; + tensor clip_33 = clip(alpha = var_1389, beta = const_20, x = s_t_9)[name = tensor("clip_33")]; tensor sqrt_s_t_9 = sqrt(x = clip_33)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_2135)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_2127)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1491)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1483)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_2153 = const()[name = tensor("op_2153"), val = tensor([1, 4])]; - tensor var_2154 = reshape(shape = var_2153, x = valid_mask)[name = tensor("op_2154")]; - tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_2154)[name = tensor("causal_with_valid_1")]; - tensor var_2156 = const()[name = tensor("op_2156"), val = tensor([4, 1])]; - tensor var_2157 = reshape(shape = var_2156, x = sqrt_s_t_9)[name = tensor("op_2157")]; - tensor M_9 = real_div(x = causal_with_valid_1, y = var_2157)[name = tensor("M_9")]; - tensor var_2159 = mul(x = qk_9, y = M_9)[name = tensor("op_2159")]; + tensor var_1509 = const()[name = tensor("op_1509"), val = tensor([1, 4])]; + tensor var_1510 = reshape(shape = var_1509, x = valid_mask)[name = tensor("op_1510")]; + tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1510)[name = tensor("causal_with_valid_1")]; + tensor var_1512 = const()[name = tensor("op_1512"), val = tensor([4, 1])]; + tensor var_1513 = reshape(shape = var_1512, x = sqrt_s_t_9)[name = tensor("op_1513")]; + tensor M_9 = real_div(x = causal_with_valid_1, y = var_1513)[name = tensor("M_9")]; + tensor var_1515 = mul(x = qk_9, y = M_9)[name = tensor("op_1515")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_2141)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_2159, y = v_9)[name = tensor("inner_9")]; - tensor var_2161_transpose_x_0 = const()[name = tensor("op_2161_transpose_x_0"), val = tensor(false)]; - tensor var_2161_transpose_y_0 = const()[name = tensor("op_2161_transpose_y_0"), val = tensor(false)]; - tensor var_2161 = matmul(transpose_x = var_2161_transpose_x_0, transpose_y = var_2161_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_2161")]; - tensor var_2162 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_2162")]; - tensor var_2163 = const()[name = tensor("op_2163"), val = tensor([1, 1, 4, 1])]; - tensor var_2164 = reshape(shape = var_2163, x = var_2162)[name = tensor("op_2164")]; - tensor cross_9 = mul(x = var_2161, y = var_2164)[name = tensor("cross_9")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1497)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1515, y = v_9)[name = tensor("inner_9")]; + tensor var_1517_transpose_x_0 = const()[name = tensor("op_1517_transpose_x_0"), val = tensor(false)]; + tensor var_1517_transpose_y_0 = const()[name = tensor("op_1517_transpose_y_0"), val = tensor(false)]; + tensor var_1517 = matmul(transpose_x = var_1517_transpose_x_0, transpose_y = var_1517_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1517")]; + tensor var_1518 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1518")]; + tensor var_1519 = const()[name = tensor("op_1519"), val = tensor([1, 1, 4, 1])]; + tensor var_1520 = reshape(shape = var_1519, x = var_1518)[name = tensor("op_1520")]; + tensor cross_9 = mul(x = var_1517, y = var_1520)[name = tensor("cross_9")]; tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; - tensor var_2167 = const()[name = tensor("op_2167"), val = tensor([1, 1, 4, 1])]; - tensor var_2168 = reshape(shape = var_2167, x = valid_mask)[name = tensor("op_2168")]; - tensor v_masked_1 = mul(x = v_9, y = var_2168)[name = tensor("v_masked_1")]; - tensor var_2170 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_2170")]; - tensor var_2172_transpose_x_1 = const()[name = tensor("op_2172_transpose_x_1"), val = tensor(true)]; - tensor var_2172_transpose_y_1 = const()[name = tensor("op_2172_transpose_y_1"), val = tensor(false)]; - tensor var_2172 = matmul(transpose_x = var_2172_transpose_x_1, transpose_y = var_2172_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_2172")]; - tensor new_kv_unnorm_9 = add(x = var_2170, y = var_2172)[name = tensor("new_kv_unnorm_9")]; - tensor var_2174_keep_dims_0 = const()[name = tensor("op_2174_keep_dims_0"), val = tensor(false)]; - tensor var_2174 = reduce_sum(keep_dims = var_2174_keep_dims_0, x = valid_mask)[name = tensor("op_2174")]; - tensor var_2175 = const()[name = tensor("op_2175"), val = tensor([1])]; - tensor var_2176 = reshape(shape = var_2175, x = var_2174)[name = tensor("op_2176")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_2176)[name = tensor("new_scale_9")]; - tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; - tensor clip_34 = clip(alpha = var_2034, beta = const_33, x = new_scale_9)[name = tensor("clip_34")]; + tensor var_1523 = const()[name = tensor("op_1523"), val = tensor([1, 1, 4, 1])]; + tensor var_1524 = reshape(shape = var_1523, x = valid_mask)[name = tensor("op_1524")]; + tensor v_masked_1 = mul(x = v_9, y = var_1524)[name = tensor("v_masked_1")]; + tensor var_1526 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1526")]; + tensor var_1528_transpose_x_1 = const()[name = tensor("op_1528_transpose_x_1"), val = tensor(true)]; + tensor var_1528_transpose_y_1 = const()[name = tensor("op_1528_transpose_y_1"), val = tensor(false)]; + tensor var_1528 = matmul(transpose_x = var_1528_transpose_x_1, transpose_y = var_1528_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1528")]; + tensor new_kv_unnorm_9 = add(x = var_1526, y = var_1528)[name = tensor("new_kv_unnorm_9")]; + tensor var_1530_keep_dims_0 = const()[name = tensor("op_1530_keep_dims_0"), val = tensor(false)]; + tensor var_1530 = reduce_sum(keep_dims = var_1530_keep_dims_0, x = valid_mask)[name = tensor("op_1530")]; + tensor var_1531 = const()[name = tensor("op_1531"), val = tensor([1])]; + tensor var_1532 = reshape(shape = var_1531, x = var_1530)[name = tensor("op_1532")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1532)[name = tensor("new_scale_9")]; + tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; + tensor clip_34 = clip(alpha = var_1389, beta = const_21, x = new_scale_9)[name = tensor("clip_34")]; tensor sqrt_new_scale_1 = sqrt(x = clip_34)[name = tensor("sqrt_new_scale_1")]; - tensor var_2180 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_2180")]; - tensor var_2181_perm_0 = const()[name = tensor("op_2181_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1536 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1536")]; + tensor var_1537_perm_0 = const()[name = tensor("op_1537_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_2181 = transpose(perm = var_2181_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_2031, x = var_2181)[name = tensor("out_27")]; - tensor var_2185 = const()[name = tensor("op_2185"), val = tensor([6, 4, 256])]; - tensor out_29 = reshape(shape = var_2185, x = out_27)[name = tensor("out_29")]; - tensor var_2187 = silu(x = input_289)[name = tensor("op_2187")]; - tensor input_291 = mul(x = var_2187, y = out_29)[name = tensor("input_291")]; - tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_291)[name = tensor("linear_42")]; - tensor input_293 = add(x = x_53, y = ret_out_9)[name = tensor("input_293")]; + tensor var_1537 = transpose(perm = var_1537_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1386, x = var_1537)[name = tensor("out_27")]; + tensor var_1541 = const()[name = tensor("op_1541"), val = tensor([6, 4, 256])]; + tensor out_29 = reshape(shape = var_1541, x = out_27)[name = tensor("out_29")]; + tensor var_1543 = silu(x = input_169)[name = tensor("op_1543")]; + tensor input_171 = mul(x = var_1543, y = out_29)[name = tensor("input_171")]; + tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; + tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_2029, gamma = decoder_norm11_0_weight, x = input_293)[name = tensor("xt_1")]; - tensor var_2197 = const()[name = tensor("op_2197"), val = tensor([1, 6, 4, 256])]; - tensor var_2198 = reshape(shape = var_2197, x = xt_1)[name = tensor("op_2198")]; - tensor var_2199_perm_0 = const()[name = tensor("op_2199_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2202 = const()[name = tensor("op_2202"), val = tensor([4, 6, 256])]; - tensor var_2199 = transpose(perm = var_2199_perm_0, x = var_2198)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_2202, x = var_2199)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1384, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1553 = const()[name = tensor("op_1553"), val = tensor([1, 6, 4, 256])]; + tensor var_1554 = reshape(shape = var_1553, x = xt_1)[name = tensor("op_1554")]; + tensor var_1555_perm_0 = const()[name = tensor("op_1555_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1558 = const()[name = tensor("op_1558"), val = tensor([4, 6, 256])]; + tensor var_1555 = transpose(perm = var_1555_perm_0, x = var_1554)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1558, x = var_1555)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_2225 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_16 = const()[name = tensor("concat_16"), val = tensor([6, 4, 3, 256])]; - tensor var_2227 = reshape(shape = concat_16, x = var_2225)[name = tensor("op_2227")]; - tensor var_2228_axes_0 = const()[name = tensor("op_2228_axes_0"), val = tensor([0])]; - tensor var_2228 = expand_dims(axes = var_2228_axes_0, x = var_2227)[name = tensor("op_2228")]; - tensor var_2229_perm_0 = const()[name = tensor("op_2229_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2230_axes_0 = const()[name = tensor("op_2230_axes_0"), val = tensor([-2])]; - tensor var_2229 = transpose(perm = var_2229_perm_0, x = var_2228)[name = tensor("transpose_21")]; - tensor var_2230 = squeeze(axes = var_2230_axes_0, x = var_2229)[name = tensor("op_2230")]; + tensor var_1581 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_17 = const()[name = tensor("concat_17"), val = tensor([6, 4, 3, 256])]; + tensor var_1583 = reshape(shape = concat_17, x = var_1581)[name = tensor("op_1583")]; + tensor var_1584_axes_0 = const()[name = tensor("op_1584_axes_0"), val = tensor([0])]; + tensor var_1584 = expand_dims(axes = var_1584_axes_0, x = var_1583)[name = tensor("op_1584")]; + tensor var_1585_perm_0 = const()[name = tensor("op_1585_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1586_axes_0 = const()[name = tensor("op_1586_axes_0"), val = tensor([-2])]; + tensor var_1585 = transpose(perm = var_1585_perm_0, x = var_1584)[name = tensor("transpose_21")]; + tensor var_1586 = squeeze(axes = var_1586_axes_0, x = var_1585)[name = tensor("op_1586")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 6, 4, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_2230)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1586)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 6, 4, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_2230)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1586)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 6, 4, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_2230)[name = tensor("v_11")]; - tensor var_2238 = const()[name = tensor("op_2238"), val = tensor([6, 16, 64])]; - tensor var_2239 = reshape(shape = var_2238, x = q_11)[name = tensor("op_2239")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1586)[name = tensor("v_11")]; + tensor var_1594 = const()[name = tensor("op_1594"), val = tensor([6, 16, 64])]; + tensor var_1595 = reshape(shape = var_1594, x = q_11)[name = tensor("op_1595")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2245 = const()[name = tensor("op_2245"), val = tensor([6, 16, 64])]; - tensor var_2246 = reshape(shape = var_2245, x = k_11)[name = tensor("op_2246")]; + tensor var_1601 = const()[name = tensor("op_1601"), val = tensor([6, 16, 64])]; + tensor var_1602 = reshape(shape = var_1601, x = k_11)[name = tensor("op_1602")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2252 = const()[name = tensor("op_2252"), val = tensor([6, 16, 64])]; - tensor var_2253 = reshape(shape = var_2252, x = v_11)[name = tensor("op_2253")]; + tensor var_1608 = const()[name = tensor("op_1608"), val = tensor([6, 16, 64])]; + tensor var_1609 = reshape(shape = var_1608, x = v_11)[name = tensor("op_1609")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2256 = const()[name = tensor("op_2256"), val = tensor([4, 4, 6, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_2239)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_2256, x = q_13)[name = tensor("q_15")]; - tensor var_2258 = const()[name = tensor("op_2258"), val = tensor([4, 4, 6, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_2246)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_2258, x = k_13)[name = tensor("k_15")]; - tensor var_2260 = const()[name = tensor("op_2260"), val = tensor([4, 4, 6, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_2253)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_2260, x = v_13)[name = tensor("v_15")]; + tensor var_1612 = const()[name = tensor("op_1612"), val = tensor([4, 4, 6, 64])]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1595)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1612, x = q_13)[name = tensor("q_15")]; + tensor var_1614 = const()[name = tensor("op_1614"), val = tensor([4, 4, 6, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1602)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1614, x = k_13)[name = tensor("k_15")]; + tensor var_1616 = const()[name = tensor("op_1616"), val = tensor([4, 4, 6, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1609)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1616, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -1989,30 +1530,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_2263 = const()[name = tensor("op_2263"), val = tensor([2, 0, 1, 3])]; - tensor var_2268 = const()[name = tensor("op_2268"), val = tensor([24, 256])]; - tensor var_2264 = transpose(perm = var_2263, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_2268, x = var_2264)[name = tensor("attn_output_3")]; + tensor var_1619 = const()[name = tensor("op_1619"), val = tensor([2, 0, 1, 3])]; + tensor var_1624 = const()[name = tensor("op_1624"), val = tensor([24, 256])]; + tensor var_1620 = transpose(perm = var_1619, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1624, x = var_1620)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_2272 = const()[name = tensor("op_2272"), val = tensor([6, 4, 256])]; - tensor attn_output_7 = reshape(shape = var_2272, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1628 = const()[name = tensor("op_1628"), val = tensor([6, 4, 256])]; + tensor attn_output_7 = reshape(shape = var_1628, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; - tensor input_295 = add(x = query_1, y = sa2_out_1)[name = tensor("input_295")]; - tensor input_297_axes_0 = const()[name = tensor("input_297_axes_0"), val = tensor([-1])]; - tensor input_297 = layer_norm(axes = input_297_axes_0, beta = decoder_norm21_0_bias, epsilon = var_2029, gamma = decoder_norm21_0_weight, x = input_295)[name = tensor("input_297")]; - tensor input_299 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_297)[name = tensor("linear_45")]; - tensor input_301 = relu(x = input_299)[name = tensor("input_301")]; - tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_301)[name = tensor("linear_46")]; - tensor input_303 = add(x = input_297, y = ffn_out_1)[name = tensor("input_303")]; + tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; + tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1384, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; + tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; + tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; + tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_2029, gamma = decoder_norm22_0_weight, x = input_303)[name = tensor("xt_3")]; - tensor var_2292 = const()[name = tensor("op_2292"), val = tensor([1, 4, 6, 256])]; - tensor x_55 = reshape(shape = var_2292, x = xt_3)[name = tensor("x_55")]; - tensor var_2294_perm_0 = const()[name = tensor("op_2294_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2298 = const()[name = tensor("op_2298"), val = tensor([6, 4, 256])]; - tensor var_2294 = transpose(perm = var_2294_perm_0, x = x_55)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_2298, x = var_2294)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1384, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1648 = const()[name = tensor("op_1648"), val = tensor([1, 4, 6, 256])]; + tensor x_31 = reshape(shape = var_1648, x = xt_3)[name = tensor("x_31")]; + tensor var_1650_perm_0 = const()[name = tensor("op_1650_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1654 = const()[name = tensor("op_1654"), val = tensor([6, 4, 256])]; + tensor var_1650 = transpose(perm = var_1650_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1654, x = var_1650)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 6, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -2023,120 +1564,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_2306 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_2307 = const()[name = tensor("op_2307"), val = tensor([6, 4, 4, 64])]; - tensor var_2308 = reshape(shape = var_2307, x = var_2306)[name = tensor("op_2308")]; + tensor var_1662 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1663 = const()[name = tensor("op_1663"), val = tensor([6, 4, 4, 64])]; + tensor var_1664 = reshape(shape = var_1663, x = var_1662)[name = tensor("op_1664")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2312 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_2313 = const()[name = tensor("op_2313"), val = tensor(0x1p-3)]; - tensor var_2314 = mul(x = var_2312, y = var_2313)[name = tensor("op_2314")]; - tensor var_2315 = const()[name = tensor("op_2315"), val = tensor([6, 4, 4, 64])]; - tensor var_2316 = reshape(shape = var_2315, x = var_2314)[name = tensor("op_2316")]; + tensor var_1668 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1669 = const()[name = tensor("op_1669"), val = tensor(0x1p-3)]; + tensor var_1670 = mul(x = var_1668, y = var_1669)[name = tensor("op_1670")]; + tensor var_1671 = const()[name = tensor("op_1671"), val = tensor([6, 4, 4, 64])]; + tensor var_1672 = reshape(shape = var_1671, x = var_1670)[name = tensor("op_1672")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2320 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_2321 = const()[name = tensor("op_2321"), val = tensor([6, 4, 4, 64])]; - tensor var_2322 = reshape(shape = var_2321, x = var_2320)[name = tensor("op_2322")]; + tensor var_1676 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1677 = const()[name = tensor("op_1677"), val = tensor([6, 4, 4, 64])]; + tensor var_1678 = reshape(shape = var_1677, x = var_1676)[name = tensor("op_1678")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_307 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; + tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; - tensor const_44 = const()[name = tensor("const_44"), val = tensor(0x1.fffffep+127)]; - tensor clip_35 = clip(alpha = var_2034, beta = const_44, x = s_t)[name = tensor("clip_35")]; + tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; + tensor clip_35 = clip(alpha = var_1389, beta = const_32, x = s_t)[name = tensor("clip_35")]; tensor sqrt_s_t = sqrt(x = clip_35)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_2316)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_2308)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1672)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1664)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_2337 = const()[name = tensor("op_2337"), val = tensor([4, 1])]; - tensor var_2338 = reshape(shape = var_2337, x = sqrt_s_t)[name = tensor("op_2338")]; - tensor M = real_div(x = causal_with_valid_1, y = var_2338)[name = tensor("M")]; - tensor var_2340 = mul(x = qk, y = M)[name = tensor("op_2340")]; + tensor var_1693 = const()[name = tensor("op_1693"), val = tensor([4, 1])]; + tensor var_1694 = reshape(shape = var_1693, x = sqrt_s_t)[name = tensor("op_1694")]; + tensor M = real_div(x = causal_with_valid_1, y = var_1694)[name = tensor("M")]; + tensor var_1696 = mul(x = qk, y = M)[name = tensor("op_1696")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_2322)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_2340, y = v_17)[name = tensor("inner")]; - tensor var_2342_transpose_x_0 = const()[name = tensor("op_2342_transpose_x_0"), val = tensor(false)]; - tensor var_2342_transpose_y_0 = const()[name = tensor("op_2342_transpose_y_0"), val = tensor(false)]; - tensor var_2342 = matmul(transpose_x = var_2342_transpose_x_0, transpose_y = var_2342_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_2342")]; - tensor var_2343 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_2343")]; - tensor var_2344 = const()[name = tensor("op_2344"), val = tensor([1, 1, 4, 1])]; - tensor var_2345 = reshape(shape = var_2344, x = var_2343)[name = tensor("op_2345")]; - tensor cross = mul(x = var_2342, y = var_2345)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1678)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1696, y = v_17)[name = tensor("inner")]; + tensor var_1698_transpose_x_0 = const()[name = tensor("op_1698_transpose_x_0"), val = tensor(false)]; + tensor var_1698_transpose_y_0 = const()[name = tensor("op_1698_transpose_y_0"), val = tensor(false)]; + tensor var_1698 = matmul(transpose_x = var_1698_transpose_x_0, transpose_y = var_1698_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1698")]; + tensor var_1699 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1699")]; + tensor var_1700 = const()[name = tensor("op_1700"), val = tensor([1, 1, 4, 1])]; + tensor var_1701 = reshape(shape = var_1700, x = var_1699)[name = tensor("op_1701")]; + tensor cross = mul(x = var_1698, y = var_1701)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_2168)[name = tensor("v_masked")]; - tensor var_2351 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_2351")]; - tensor var_2353_transpose_x_1 = const()[name = tensor("op_2353_transpose_x_1"), val = tensor(true)]; - tensor var_2353_transpose_y_1 = const()[name = tensor("op_2353_transpose_y_1"), val = tensor(false)]; - tensor var_2353 = matmul(transpose_x = var_2353_transpose_x_1, transpose_y = var_2353_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_2353")]; - tensor new_kv_unnorm = add(x = var_2351, y = var_2353)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_2176)[name = tensor("new_scale")]; - tensor const_45 = const()[name = tensor("const_45"), val = tensor(0x1.fffffep+127)]; - tensor clip_36 = clip(alpha = var_2034, beta = const_45, x = new_scale)[name = tensor("clip_36")]; + tensor v_masked = mul(x = v_17, y = var_1524)[name = tensor("v_masked")]; + tensor var_1707 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1707")]; + tensor var_1709_transpose_x_1 = const()[name = tensor("op_1709_transpose_x_1"), val = tensor(true)]; + tensor var_1709_transpose_y_1 = const()[name = tensor("op_1709_transpose_y_1"), val = tensor(false)]; + tensor var_1709 = matmul(transpose_x = var_1709_transpose_x_1, transpose_y = var_1709_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1709")]; + tensor new_kv_unnorm = add(x = var_1707, y = var_1709)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1532)[name = tensor("new_scale")]; + tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; + tensor clip_36 = clip(alpha = var_1389, beta = const_33, x = new_scale)[name = tensor("clip_36")]; tensor sqrt_new_scale = sqrt(x = clip_36)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_2362_perm_0 = const()[name = tensor("op_2362_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1718_perm_0 = const()[name = tensor("op_1718_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_2362 = transpose(perm = var_2362_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_2031, x = var_2362)[name = tensor("out_33")]; - tensor var_2366 = const()[name = tensor("op_2366"), val = tensor([6, 4, 256])]; - tensor out = reshape(shape = var_2366, x = out_33)[name = tensor("out")]; - tensor var_2368 = silu(x = input_307)[name = tensor("op_2368")]; - tensor input_309 = mul(x = var_2368, y = out)[name = tensor("input_309")]; - tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_309)[name = tensor("linear_51")]; - tensor input_311 = add(x = x, y = ret_out)[name = tensor("input_311")]; + tensor var_1718 = transpose(perm = var_1718_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1386, x = var_1718)[name = tensor("out_33")]; + tensor var_1722 = const()[name = tensor("op_1722"), val = tensor([6, 4, 256])]; + tensor out = reshape(shape = var_1722, x = out_33)[name = tensor("out")]; + tensor var_1724 = silu(x = input_187)[name = tensor("op_1724")]; + tensor input_189 = mul(x = var_1724, y = out)[name = tensor("input_189")]; + tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; + tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_2029, gamma = decoder_norm11_1_weight, x = input_311)[name = tensor("xt_5")]; - tensor var_2378 = const()[name = tensor("op_2378"), val = tensor([1, 6, 4, 256])]; - tensor var_2379 = reshape(shape = var_2378, x = xt_5)[name = tensor("op_2379")]; - tensor var_2380_perm_0 = const()[name = tensor("op_2380_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2383 = const()[name = tensor("op_2383"), val = tensor([4, 6, 256])]; - tensor var_2380 = transpose(perm = var_2380_perm_0, x = var_2379)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_2383, x = var_2380)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1384, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1734 = const()[name = tensor("op_1734"), val = tensor([1, 6, 4, 256])]; + tensor var_1735 = reshape(shape = var_1734, x = xt_5)[name = tensor("op_1735")]; + tensor var_1736_perm_0 = const()[name = tensor("op_1736_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1739 = const()[name = tensor("op_1739"), val = tensor([4, 6, 256])]; + tensor var_1736 = transpose(perm = var_1736_perm_0, x = var_1735)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1739, x = var_1736)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_2406 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_17 = const()[name = tensor("concat_17"), val = tensor([6, 4, 3, 256])]; - tensor var_2408 = reshape(shape = concat_17, x = var_2406)[name = tensor("op_2408")]; - tensor var_2409_axes_0 = const()[name = tensor("op_2409_axes_0"), val = tensor([0])]; - tensor var_2409 = expand_dims(axes = var_2409_axes_0, x = var_2408)[name = tensor("op_2409")]; - tensor var_2410_perm_0 = const()[name = tensor("op_2410_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2411_axes_0 = const()[name = tensor("op_2411_axes_0"), val = tensor([-2])]; - tensor var_2410 = transpose(perm = var_2410_perm_0, x = var_2409)[name = tensor("transpose_8")]; - tensor var_2411 = squeeze(axes = var_2411_axes_0, x = var_2410)[name = tensor("op_2411")]; + tensor var_1762 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_18 = const()[name = tensor("concat_18"), val = tensor([6, 4, 3, 256])]; + tensor var_1764 = reshape(shape = concat_18, x = var_1762)[name = tensor("op_1764")]; + tensor var_1765_axes_0 = const()[name = tensor("op_1765_axes_0"), val = tensor([0])]; + tensor var_1765 = expand_dims(axes = var_1765_axes_0, x = var_1764)[name = tensor("op_1765")]; + tensor var_1766_perm_0 = const()[name = tensor("op_1766_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1767_axes_0 = const()[name = tensor("op_1767_axes_0"), val = tensor([-2])]; + tensor var_1766 = transpose(perm = var_1766_perm_0, x = var_1765)[name = tensor("transpose_8")]; + tensor var_1767 = squeeze(axes = var_1767_axes_0, x = var_1766)[name = tensor("op_1767")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 6, 4, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_2411)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1767)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 6, 4, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_2411)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1767)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 6, 4, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_2411)[name = tensor("v_19")]; - tensor var_2419 = const()[name = tensor("op_2419"), val = tensor([6, 16, 64])]; - tensor var_2420 = reshape(shape = var_2419, x = q_19)[name = tensor("op_2420")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1767)[name = tensor("v_19")]; + tensor var_1775 = const()[name = tensor("op_1775"), val = tensor([6, 16, 64])]; + tensor var_1776 = reshape(shape = var_1775, x = q_19)[name = tensor("op_1776")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2426 = const()[name = tensor("op_2426"), val = tensor([6, 16, 64])]; - tensor var_2427 = reshape(shape = var_2426, x = k_19)[name = tensor("op_2427")]; + tensor var_1782 = const()[name = tensor("op_1782"), val = tensor([6, 16, 64])]; + tensor var_1783 = reshape(shape = var_1782, x = k_19)[name = tensor("op_1783")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2433 = const()[name = tensor("op_2433"), val = tensor([6, 16, 64])]; - tensor var_2434 = reshape(shape = var_2433, x = v_19)[name = tensor("op_2434")]; + tensor var_1789 = const()[name = tensor("op_1789"), val = tensor([6, 16, 64])]; + tensor var_1790 = reshape(shape = var_1789, x = v_19)[name = tensor("op_1790")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2437 = const()[name = tensor("op_2437"), val = tensor([4, 4, 6, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_2420)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_2437, x = q_21)[name = tensor("q")]; - tensor var_2439 = const()[name = tensor("op_2439"), val = tensor([4, 4, 6, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_2427)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_2439, x = k_21)[name = tensor("k")]; - tensor var_2441 = const()[name = tensor("op_2441"), val = tensor([4, 4, 6, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_2434)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_2441, x = v_21)[name = tensor("v")]; + tensor var_1793 = const()[name = tensor("op_1793"), val = tensor([4, 4, 6, 64])]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1776)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1793, x = q_21)[name = tensor("q")]; + tensor var_1795 = const()[name = tensor("op_1795"), val = tensor([4, 4, 6, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1783)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1795, x = k_21)[name = tensor("k")]; + tensor var_1797 = const()[name = tensor("op_1797"), val = tensor([4, 4, 6, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1790)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1797, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -2147,50 +1688,50 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_2444 = const()[name = tensor("op_2444"), val = tensor([2, 0, 1, 3])]; - tensor var_2449 = const()[name = tensor("op_2449"), val = tensor([24, 256])]; - tensor var_2445 = transpose(perm = var_2444, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_2449, x = var_2445)[name = tensor("attn_output_11")]; + tensor var_1800 = const()[name = tensor("op_1800"), val = tensor([2, 0, 1, 3])]; + tensor var_1805 = const()[name = tensor("op_1805"), val = tensor([24, 256])]; + tensor var_1801 = transpose(perm = var_1800, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1805, x = var_1801)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_2453 = const()[name = tensor("op_2453"), val = tensor([6, 4, 256])]; - tensor attn_output = reshape(shape = var_2453, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1809 = const()[name = tensor("op_1809"), val = tensor([6, 4, 256])]; + tensor attn_output = reshape(shape = var_1809, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; - tensor input_313 = add(x = query_5, y = sa2_out)[name = tensor("input_313")]; - tensor input_315_axes_0 = const()[name = tensor("input_315_axes_0"), val = tensor([-1])]; - tensor input_315 = layer_norm(axes = input_315_axes_0, beta = decoder_norm21_1_bias, epsilon = var_2029, gamma = decoder_norm21_1_weight, x = input_313)[name = tensor("input_315")]; - tensor input_317 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_315)[name = tensor("linear_54")]; - tensor input_319 = relu(x = input_317)[name = tensor("input_319")]; - tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_319)[name = tensor("linear_55")]; - tensor input_321 = add(x = input_315, y = ffn_out)[name = tensor("input_321")]; + tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; + tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1384, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; + tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; + tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; + tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_2029, gamma = decoder_norm22_1_weight, x = input_321)[name = tensor("xt")]; - tensor var_2473 = const()[name = tensor("op_2473"), val = tensor([1, 4, 6, 256])]; - tensor input = reshape(shape = var_2473, x = xt)[name = tensor("input")]; - tensor var_2475 = const()[name = tensor("op_2475"), val = tensor([-1])]; - tensor var_2476 = reduce_l2_norm(axes = var_2475, keep_dims = var_2032, x = input)[name = tensor("op_2476")]; - tensor const_54 = const()[name = tensor("const_54"), val = tensor(0x1.fffffep+127)]; - tensor clip_37 = clip(alpha = var_2023, beta = const_54, x = var_2476)[name = tensor("clip_37")]; - tensor var_2478 = real_div(x = input, y = clip_37)[name = tensor("op_2478")]; - tensor concat_21 = const()[name = tensor("concat_21"), val = tensor([4, 1, 256])]; - tensor reshape_0 = reshape(shape = concat_21, x = emb)[name = tensor("reshape_0")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1384, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1829 = const()[name = tensor("op_1829"), val = tensor([1, 4, 6, 256])]; + tensor input = reshape(shape = var_1829, x = xt)[name = tensor("input")]; + tensor var_1831 = const()[name = tensor("op_1831"), val = tensor([-1])]; + tensor var_1832 = reduce_l2_norm(axes = var_1831, keep_dims = var_1387, x = input)[name = tensor("op_1832")]; + tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; + tensor clip_37 = clip(alpha = var_1379, beta = const_42, x = var_1832)[name = tensor("clip_37")]; + tensor var_1834 = real_div(x = input, y = clip_37)[name = tensor("op_1834")]; + tensor concat_22 = const()[name = tensor("concat_22"), val = tensor([4, 1, 256])]; + tensor reshape_0 = reshape(shape = concat_22, x = emb)[name = tensor("reshape_0")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_22 = const()[name = tensor("concat_22"), val = tensor([4, 256, 6])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_2478)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_22, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_23 = const()[name = tensor("concat_23"), val = tensor([4, 256, 6])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1834)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_23, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = reshape_0, y = reshape_1)[name = tensor("matmul_2")]; - tensor concat_25 = const()[name = tensor("concat_25"), val = tensor([1, 4, 6])]; - tensor reshape_2 = reshape(shape = concat_25, x = matmul_2)[name = tensor("reshape_2")]; + tensor concat_26 = const()[name = tensor("concat_26"), val = tensor([1, 4, 6])]; + tensor reshape_2 = reshape(shape = concat_26, x = matmul_2)[name = tensor("reshape_2")]; tensor output_begin_0 = const()[name = tensor("output_begin_0"), val = tensor([0, 0, 1])]; tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 4, 5])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = reshape_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_2482")]; - tensor var_2484_axis_0 = const()[name = tensor("op_2484_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_2484_axis_0, values = (var_2180, nkv))[name = tensor("op_2484")]; - tensor var_2486_axis_0 = const()[name = tensor("op_2486_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_2486_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_2486")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1838")]; + tensor var_1840_axis_0 = const()[name = tensor("op_1840_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1840_axis_0, values = (var_1536, nkv))[name = tensor("op_1840")]; + tensor var_1842_axis_0 = const()[name = tensor("op_1842_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1842_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1842")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/AMI/ls_eend_ami_400ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/AMI/ls_eend_ami_400ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index a0db65e636ed527ba560e20b90c49e4e6e75d4d5..8ae7a96b28e3f189f4b73b079cac6963ee581218 100644 --- a/optimized/AMI/ls_eend_ami_400ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/AMI/ls_eend_ami_400ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:07afe6ce7c68aea8aad72dae82ac6a50acd2317836362872b8e76619470c265f -size 311043 +oid sha256:b90301923548d2f548add977934614ec2f4b7d2db75a3d8acacbc2abab97d5a9 +size 238948 diff --git a/optimized/AMI/ls_eend_ami_400ms.mlpackage/Manifest.json b/optimized/AMI/ls_eend_ami_400ms.mlpackage/Manifest.json index 351633b83ba6a0836dca7b6464c5338eab7e8efc..c5d19ff4378bb9e5d5c8feafb5d75bcc13ae30d9 100644 --- a/optimized/AMI/ls_eend_ami_400ms.mlpackage/Manifest.json +++ b/optimized/AMI/ls_eend_ami_400ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "50484F5C-DB73-4CB4-B7D9-9573B42D9900": { + "24FD3DCB-49E9-4D6A-A75A-AC255784E274": { "author": "com.apple.CoreML", "description": "CoreML Model Specification", "name": "model.mlmodel", "path": "com.apple.CoreML/model.mlmodel" }, - "550C96F7-965E-447D-BB53-2C25F9F0542E": { + "E887992C-463A-4692-AC5B-A64A28CD7A91": { "author": "com.apple.CoreML", "description": "CoreML Model Weights", "name": "weights", "path": "com.apple.CoreML/weights" } }, - "rootModelIdentifier": "50484F5C-DB73-4CB4-B7D9-9573B42D9900" + "rootModelIdentifier": "24FD3DCB-49E9-4D6A-A75A-AC255784E274" } diff --git a/optimized/AMI/ls_eend_ami_500ms.mlmodelc/analytics/coremldata.bin b/optimized/AMI/ls_eend_ami_500ms.mlmodelc/analytics/coremldata.bin index 6e8066e749dc010b8d0c897e5ce7804dfc69d9c6..71cd7de07049e231c6742e501c7a536142d22101 100644 --- a/optimized/AMI/ls_eend_ami_500ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/AMI/ls_eend_ami_500ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b1926f15595385ff86f871a3d197974e55a1dd5ab6eff7a1b12ead1ce1da34d6 +oid sha256:8caf9dda832742c0a1a5acd9d7b9b6fd4fa1d07bb031cf7f6e0acfccd1f467a4 size 243 diff --git a/optimized/AMI/ls_eend_ami_500ms.mlmodelc/coremldata.bin b/optimized/AMI/ls_eend_ami_500ms.mlmodelc/coremldata.bin index 6dc97bd3f75db4d1cdde7b6f3c1a0ed222d711d6..10eb7f21ab085637e7479e44d00e2056d0015880 100644 --- a/optimized/AMI/ls_eend_ami_500ms.mlmodelc/coremldata.bin +++ b/optimized/AMI/ls_eend_ami_500ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8b85b015c423dc62c5fee11df086612b4efa3b6481befb0670a8536ef12eac8b -size 1250 +oid sha256:01f4528518a5cd3a91873a81c0c2c335bfefb4789de4584b0b1334918fd5e526 +size 1341 diff --git a/optimized/AMI/ls_eend_ami_500ms.mlmodelc/metadata.json b/optimized/AMI/ls_eend_ami_500ms.mlmodelc/metadata.json index 4350bb5996ed467e21f6b7c04f9344d9d4232727..55c35eee083db6bca206a5977c1de3b861253e63 100644 --- a/optimized/AMI/ls_eend_ami_500ms.mlmodelc/metadata.json +++ b/optimized/AMI/ls_eend_ami_500ms.mlmodelc/metadata.json @@ -96,30 +96,30 @@ "Ios17.reshape" : 108, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 85, - "Split" : 20, + "Ios17.transpose" : 57, + "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 106, "Tile" : 21, - "Ios17.sliceByIndex" : 108, - "Ios16.sigmoid" : 21, + "Ios17.sliceByIndex" : 76, + "Ios16.sigmoid" : 5, "Ios16.reduceSum" : 1, "Ios17.squeeze" : 2, - "Ios17.layerNorm" : 49, - "Ios17.batchNorm" : 20, + "Ios17.layerNorm" : 33, + "Ios17.batchNorm" : 4, "Ios17.cast" : 40, "Ios17.sqrt" : 18, "Ios17.sub" : 40, - "Ios17.conv" : 61, + "Ios17.conv" : 13, "Ios17.realDiv" : 20, "Ios17.linear" : 56, "Ios17.concat" : 26, "Ios17.clip" : 46, "Ios17.greaterEqual" : 20, - "Ios16.silu" : 34, + "Ios16.silu" : 18, "Stack" : 6, "Ios16.cumsum" : 1, - "Ios17.mul" : 143 + "Ios17.mul" : 127 }, "computePrecision" : "Mixed (Float32, Int32)", "isUpdatable" : "0", @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"ami\", \"model_label\": \"AMI\", \"variant\": \"pipeline\", \"T\": 5, \"max_speakers\": 4, \"max_nspks\": 6, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"ami\", \"model_label\": \"AMI\", \"variant\": \"pipeline\", \"chunk_size\": 5, \"step_duration_ms\": 500, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 4, \"max_nspks\": 6, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/AMI/ls_eend_ami_500ms.mlmodelc/model.mil b/optimized/AMI/ls_eend_ami_500ms.mlmodelc/model.mil index dbde4f6a7907a5f4171f179b05ed6e3681a7acc1..50edd081a3decf220dbe8cde6645f6e959b34f39 100644 --- a/optimized/AMI/ls_eend_ami_500ms.mlmodelc/model.mil +++ b/optimized/AMI/ls_eend_ami_500ms.mlmodelc/model.mil @@ -217,6 +217,7 @@ program(1.0) tensor var_22 = const()[name = tensor("op_22"), val = tensor(0x0p+0)]; tensor var_25 = const()[name = tensor("op_25"), val = tensor(0x1.0c6f7ap-20)]; tensor var_28 = const()[name = tensor("op_28"), val = tensor(2)]; + tensor var_31 = const()[name = tensor("op_31"), val = tensor(0)]; tensor var_34 = const()[name = tensor("op_34"), val = tensor(1)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(0x1.4f8b58p-17)]; tensor var_37 = const()[name = tensor("op_37"), val = tensor(true)]; @@ -263,8 +264,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_181)[name = tensor("transpose_85")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_173)[name = tensor("transpose_86")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_181)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_173)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_197 = const()[name = tensor("op_197"), val = tensor([5, 1])]; tensor var_198 = reshape(shape = var_197, x = sqrt_s_t_1)[name = tensor("op_198")]; @@ -272,7 +273,7 @@ program(1.0) tensor var_200 = mul(x = qk_1, y = M_1)[name = tensor("op_200")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_187)[name = tensor("transpose_84")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_187)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_200, y = v_1)[name = tensor("inner_1")]; tensor var_202_transpose_x_0 = const()[name = tensor("op_202_transpose_x_0"), val = tensor(false)]; tensor var_202_transpose_y_0 = const()[name = tensor("op_202_transpose_y_0"), val = tensor(false)]; @@ -293,7 +294,7 @@ program(1.0) tensor var_215 = real_div(x = new_kv_unnorm_1, y = var_214)[name = tensor("op_215")]; tensor var_216_perm_0 = const()[name = tensor("op_216_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_216 = transpose(perm = var_216_perm_0, x = out_1)[name = tensor("transpose_83")]; + tensor var_216 = transpose(perm = var_216_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_25, x = var_216)[name = tensor("out_3")]; tensor var_220 = const()[name = tensor("op_220"), val = tensor([1, 5, 256])]; tensor out_5 = reshape(shape = var_220, x = out_3)[name = tensor("out_5")]; @@ -341,364 +342,210 @@ program(1.0) tensor var_253 = sub(x = var_18, y = use_roll_1)[name = tensor("op_253")]; tensor var_254 = mul(x = filled_window_1, y = var_253)[name = tensor("op_254")]; tensor var_255 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_255")]; - tensor input_21 = add(x = var_254, y = var_255)[name = tensor("input_21")]; + tensor window_3 = add(x = var_254, y = var_255)[name = tensor("window_3")]; tensor var_257 = const()[name = tensor("op_257"), val = tensor(0x1p+0)]; tensor var_258 = add(x = filled_1, y = var_257)[name = tensor("op_258")]; tensor clip_1 = clip(alpha = var_22, beta = var_17, x = var_258)[name = tensor("clip_1")]; + tensor var_261_begin_0 = const()[name = tensor("op_261_begin_0"), val = tensor([0, 1, 0])]; + tensor var_261_end_0 = const()[name = tensor("op_261_end_0"), val = tensor([1, 2, 256])]; + tensor var_261_end_mask_0 = const()[name = tensor("op_261_end_mask_0"), val = tensor([true, false, true])]; + tensor var_261 = slice_by_index(begin = var_261_begin_0, end = var_261_end_0, end_mask = var_261_end_mask_0, x = x_3)[name = tensor("op_261")]; + tensor var_264_begin_0 = const()[name = tensor("op_264_begin_0"), val = tensor([0, 1, 0])]; + tensor var_264_end_0 = const()[name = tensor("op_264_end_0"), val = tensor([1, 16, 256])]; + tensor var_264_end_mask_0 = const()[name = tensor("op_264_end_mask_0"), val = tensor([true, true, true])]; + tensor var_264 = slice_by_index(begin = var_264_begin_0, end = var_264_end_0, end_mask = var_264_end_mask_0, x = window_3)[name = tensor("op_264")]; + tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; + tensor rolled_3 = concat(axis = var_34, interleave = rolled_3_interleave_0, values = (var_264, var_261))[name = tensor("rolled_3")]; + tensor clip_2 = clip(alpha = var_22, beta = var_21, x = clip_1)[name = tensor("clip_2")]; + tensor var_269 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_269")]; + tensor var_270_dtype_0 = const()[name = tensor("op_270_dtype_0"), val = tensor("fp32")]; + tensor var_271 = const()[name = tensor("op_271"), val = tensor([1, 16, 1])]; + tensor var_270 = cast(dtype = var_270_dtype_0, x = var_269)[name = tensor("cast_189")]; + tensor fill_mask_3 = reshape(shape = var_271, x = var_270)[name = tensor("fill_mask_3")]; + tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_261)[name = tensor("x_expanded_3")]; + tensor var_275 = sub(x = var_18, y = fill_mask_3)[name = tensor("op_275")]; + tensor var_276 = mul(x = window_3, y = var_275)[name = tensor("op_276")]; + tensor var_277 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_277")]; + tensor filled_window_3 = add(x = var_276, y = var_277)[name = tensor("filled_window_3")]; + tensor var_279 = greater_equal(x = clip_1, y = var_17)[name = tensor("op_279")]; + tensor var_280_dtype_0 = const()[name = tensor("op_280_dtype_0"), val = tensor("fp32")]; + tensor var_281 = const()[name = tensor("op_281"), val = tensor([1, 1, 1])]; + tensor var_280 = cast(dtype = var_280_dtype_0, x = var_279)[name = tensor("cast_188")]; + tensor use_roll_3 = reshape(shape = var_281, x = var_280)[name = tensor("use_roll_3")]; + tensor var_283 = sub(x = var_18, y = use_roll_3)[name = tensor("op_283")]; + tensor var_284 = mul(x = filled_window_3, y = var_283)[name = tensor("op_284")]; + tensor var_285 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_285")]; + tensor window_5 = add(x = var_284, y = var_285)[name = tensor("window_5")]; + tensor var_287 = const()[name = tensor("op_287"), val = tensor(0x1p+0)]; + tensor var_288 = add(x = clip_1, y = var_287)[name = tensor("op_288")]; + tensor clip_3 = clip(alpha = var_22, beta = var_17, x = var_288)[name = tensor("clip_3")]; + tensor var_291_begin_0 = const()[name = tensor("op_291_begin_0"), val = tensor([0, 2, 0])]; + tensor var_291_end_0 = const()[name = tensor("op_291_end_0"), val = tensor([1, 3, 256])]; + tensor var_291_end_mask_0 = const()[name = tensor("op_291_end_mask_0"), val = tensor([true, false, true])]; + tensor var_291 = slice_by_index(begin = var_291_begin_0, end = var_291_end_0, end_mask = var_291_end_mask_0, x = x_3)[name = tensor("op_291")]; + tensor var_294_begin_0 = const()[name = tensor("op_294_begin_0"), val = tensor([0, 1, 0])]; + tensor var_294_end_0 = const()[name = tensor("op_294_end_0"), val = tensor([1, 16, 256])]; + tensor var_294_end_mask_0 = const()[name = tensor("op_294_end_mask_0"), val = tensor([true, true, true])]; + tensor var_294 = slice_by_index(begin = var_294_begin_0, end = var_294_end_0, end_mask = var_294_end_mask_0, x = window_5)[name = tensor("op_294")]; + tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; + tensor rolled_5 = concat(axis = var_34, interleave = rolled_5_interleave_0, values = (var_294, var_291))[name = tensor("rolled_5")]; + tensor clip_4 = clip(alpha = var_22, beta = var_21, x = clip_3)[name = tensor("clip_4")]; + tensor var_299 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_299")]; + tensor var_300_dtype_0 = const()[name = tensor("op_300_dtype_0"), val = tensor("fp32")]; + tensor var_301 = const()[name = tensor("op_301"), val = tensor([1, 16, 1])]; + tensor var_300 = cast(dtype = var_300_dtype_0, x = var_299)[name = tensor("cast_187")]; + tensor fill_mask_5 = reshape(shape = var_301, x = var_300)[name = tensor("fill_mask_5")]; + tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_291)[name = tensor("x_expanded_5")]; + tensor var_305 = sub(x = var_18, y = fill_mask_5)[name = tensor("op_305")]; + tensor var_306 = mul(x = window_5, y = var_305)[name = tensor("op_306")]; + tensor var_307 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_307")]; + tensor filled_window_5 = add(x = var_306, y = var_307)[name = tensor("filled_window_5")]; + tensor var_309 = greater_equal(x = clip_3, y = var_17)[name = tensor("op_309")]; + tensor var_310_dtype_0 = const()[name = tensor("op_310_dtype_0"), val = tensor("fp32")]; + tensor var_311 = const()[name = tensor("op_311"), val = tensor([1, 1, 1])]; + tensor var_310 = cast(dtype = var_310_dtype_0, x = var_309)[name = tensor("cast_186")]; + tensor use_roll_5 = reshape(shape = var_311, x = var_310)[name = tensor("use_roll_5")]; + tensor var_313 = sub(x = var_18, y = use_roll_5)[name = tensor("op_313")]; + tensor var_314 = mul(x = filled_window_5, y = var_313)[name = tensor("op_314")]; + tensor var_315 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_315")]; + tensor window_7 = add(x = var_314, y = var_315)[name = tensor("window_7")]; + tensor var_317 = const()[name = tensor("op_317"), val = tensor(0x1p+0)]; + tensor var_318 = add(x = clip_3, y = var_317)[name = tensor("op_318")]; + tensor clip_5 = clip(alpha = var_22, beta = var_17, x = var_318)[name = tensor("clip_5")]; + tensor var_321_begin_0 = const()[name = tensor("op_321_begin_0"), val = tensor([0, 3, 0])]; + tensor var_321_end_0 = const()[name = tensor("op_321_end_0"), val = tensor([1, 4, 256])]; + tensor var_321_end_mask_0 = const()[name = tensor("op_321_end_mask_0"), val = tensor([true, false, true])]; + tensor var_321 = slice_by_index(begin = var_321_begin_0, end = var_321_end_0, end_mask = var_321_end_mask_0, x = x_3)[name = tensor("op_321")]; + tensor var_324_begin_0 = const()[name = tensor("op_324_begin_0"), val = tensor([0, 1, 0])]; + tensor var_324_end_0 = const()[name = tensor("op_324_end_0"), val = tensor([1, 16, 256])]; + tensor var_324_end_mask_0 = const()[name = tensor("op_324_end_mask_0"), val = tensor([true, true, true])]; + tensor var_324 = slice_by_index(begin = var_324_begin_0, end = var_324_end_0, end_mask = var_324_end_mask_0, x = window_7)[name = tensor("op_324")]; + tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; + tensor rolled_7 = concat(axis = var_34, interleave = rolled_7_interleave_0, values = (var_324, var_321))[name = tensor("rolled_7")]; + tensor clip_6 = clip(alpha = var_22, beta = var_21, x = clip_5)[name = tensor("clip_6")]; + tensor var_329 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_329")]; + tensor var_330_dtype_0 = const()[name = tensor("op_330_dtype_0"), val = tensor("fp32")]; + tensor var_331 = const()[name = tensor("op_331"), val = tensor([1, 16, 1])]; + tensor var_330 = cast(dtype = var_330_dtype_0, x = var_329)[name = tensor("cast_185")]; + tensor fill_mask_7 = reshape(shape = var_331, x = var_330)[name = tensor("fill_mask_7")]; + tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_321)[name = tensor("x_expanded_7")]; + tensor var_335 = sub(x = var_18, y = fill_mask_7)[name = tensor("op_335")]; + tensor var_336 = mul(x = window_7, y = var_335)[name = tensor("op_336")]; + tensor var_337 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_337")]; + tensor filled_window_7 = add(x = var_336, y = var_337)[name = tensor("filled_window_7")]; + tensor var_339 = greater_equal(x = clip_5, y = var_17)[name = tensor("op_339")]; + tensor var_340_dtype_0 = const()[name = tensor("op_340_dtype_0"), val = tensor("fp32")]; + tensor var_341 = const()[name = tensor("op_341"), val = tensor([1, 1, 1])]; + tensor var_340 = cast(dtype = var_340_dtype_0, x = var_339)[name = tensor("cast_184")]; + tensor use_roll_7 = reshape(shape = var_341, x = var_340)[name = tensor("use_roll_7")]; + tensor var_343 = sub(x = var_18, y = use_roll_7)[name = tensor("op_343")]; + tensor var_344 = mul(x = filled_window_7, y = var_343)[name = tensor("op_344")]; + tensor var_345 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_345")]; + tensor window_9 = add(x = var_344, y = var_345)[name = tensor("window_9")]; + tensor var_347 = const()[name = tensor("op_347"), val = tensor(0x1p+0)]; + tensor var_348 = add(x = clip_5, y = var_347)[name = tensor("op_348")]; + tensor clip_7 = clip(alpha = var_22, beta = var_17, x = var_348)[name = tensor("clip_7")]; + tensor var_351_begin_0 = const()[name = tensor("op_351_begin_0"), val = tensor([0, 4, 0])]; + tensor var_351_end_0 = const()[name = tensor("op_351_end_0"), val = tensor([1, 1, 256])]; + tensor var_351_end_mask_0 = const()[name = tensor("op_351_end_mask_0"), val = tensor([true, true, true])]; + tensor var_351 = slice_by_index(begin = var_351_begin_0, end = var_351_end_0, end_mask = var_351_end_mask_0, x = x_3)[name = tensor("op_351")]; + tensor var_354_begin_0 = const()[name = tensor("op_354_begin_0"), val = tensor([0, 1, 0])]; + tensor var_354_end_0 = const()[name = tensor("op_354_end_0"), val = tensor([1, 16, 256])]; + tensor var_354_end_mask_0 = const()[name = tensor("op_354_end_mask_0"), val = tensor([true, true, true])]; + tensor var_354 = slice_by_index(begin = var_354_begin_0, end = var_354_end_0, end_mask = var_354_end_mask_0, x = window_9)[name = tensor("op_354")]; + tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; + tensor rolled_9 = concat(axis = var_34, interleave = rolled_9_interleave_0, values = (var_354, var_351))[name = tensor("rolled_9")]; + tensor clip_8 = clip(alpha = var_22, beta = var_21, x = clip_7)[name = tensor("clip_8")]; + tensor var_359 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_359")]; + tensor var_360_dtype_0 = const()[name = tensor("op_360_dtype_0"), val = tensor("fp32")]; + tensor var_361 = const()[name = tensor("op_361"), val = tensor([1, 16, 1])]; + tensor var_360 = cast(dtype = var_360_dtype_0, x = var_359)[name = tensor("cast_183")]; + tensor fill_mask_9 = reshape(shape = var_361, x = var_360)[name = tensor("fill_mask_9")]; + tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_351)[name = tensor("x_expanded_9")]; + tensor var_365 = sub(x = var_18, y = fill_mask_9)[name = tensor("op_365")]; + tensor var_366 = mul(x = window_9, y = var_365)[name = tensor("op_366")]; + tensor var_367 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_367")]; + tensor filled_window_9 = add(x = var_366, y = var_367)[name = tensor("filled_window_9")]; + tensor var_369 = greater_equal(x = clip_7, y = var_17)[name = tensor("op_369")]; + tensor var_370_dtype_0 = const()[name = tensor("op_370_dtype_0"), val = tensor("fp32")]; + tensor var_371 = const()[name = tensor("op_371"), val = tensor([1, 1, 1])]; + tensor var_370 = cast(dtype = var_370_dtype_0, x = var_369)[name = tensor("cast_182")]; + tensor use_roll_9 = reshape(shape = var_371, x = var_370)[name = tensor("use_roll_9")]; + tensor var_373 = sub(x = var_18, y = use_roll_9)[name = tensor("op_373")]; + tensor var_374 = mul(x = filled_window_9, y = var_373)[name = tensor("op_374")]; + tensor var_375 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_375")]; + tensor window_11 = add(x = var_374, y = var_375)[name = tensor("window_11")]; + tensor var_377 = const()[name = tensor("op_377"), val = tensor(0x1p+0)]; + tensor var_378 = add(x = clip_7, y = var_377)[name = tensor("op_378")]; + tensor clip_9 = clip(alpha = var_22, beta = var_17, x = var_378)[name = tensor("clip_9")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_31, interleave = input_21_interleave_0, values = (window_3, window_5, window_7, window_9, window_11))[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; - tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; + tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; tensor inputs_3_pad_type_0 = const()[name = tensor("inputs_3_pad_type_0"), val = tensor("valid")]; tensor inputs_3_strides_0 = const()[name = tensor("inputs_3_strides_0"), val = tensor([1])]; tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_82")]; - tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_279_split_sizes_0 = const()[name = tensor("op_279_split_sizes_0"), val = tensor([256, 256])]; - tensor var_279_axis_0 = const()[name = tensor("op_279_axis_0"), val = tensor(1)]; - tensor var_279_0, tensor var_279_1 = split(axis = var_279_axis_0, split_sizes = var_279_split_sizes_0, x = inputs_3)[name = tensor("op_279")]; - tensor var_281 = sigmoid(x = var_279_1)[name = tensor("op_281")]; - tensor inputs_5 = mul(x = var_279_0, y = var_281)[name = tensor("inputs_5")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; + tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; + tensor var_401_split_sizes_0 = const()[name = tensor("op_401_split_sizes_0"), val = tensor([256, 256])]; + tensor var_401_axis_0 = const()[name = tensor("op_401_axis_0"), val = tensor(1)]; + tensor var_401_0, tensor var_401_1 = split(axis = var_401_axis_0, split_sizes = var_401_split_sizes_0, x = inputs_3)[name = tensor("op_401")]; + tensor var_403 = sigmoid(x = var_401_1)[name = tensor("op_403")]; + tensor inputs_5 = mul(x = var_401_0, y = var_403)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; tensor outputs_aug_1_strides_0 = const()[name = tensor("outputs_aug_1_strides_0"), val = tensor([1])]; tensor outputs_aug_1_dilations_0 = const()[name = tensor("outputs_aug_1_dilations_0"), val = tensor([1])]; - tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; + tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; tensor input_25_begin_0 = const()[name = tensor("input_25_begin_0"), val = tensor([0, 0, 0])]; - tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([1, 256, 16])]; + tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([5, 256, 16])]; tensor input_25_end_mask_0 = const()[name = tensor("input_25_end_mask_0"), val = tensor([true, true, false])]; - tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; - tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; - tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; + tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; + tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; + tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; tensor input_29_pad_type_0 = const()[name = tensor("input_29_pad_type_0"), val = tensor("valid")]; tensor input_29_strides_0 = const()[name = tensor("input_29_strides_0"), val = tensor([1])]; tensor input_29_pad_0 = const()[name = tensor("input_29_pad_0"), val = tensor([0, 0])]; tensor input_29_dilations_0 = const()[name = tensor("input_29_dilations_0"), val = tensor([1])]; tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; - tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; + tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_312_begin_0 = const()[name = tensor("op_312_begin_0"), val = tensor([0, -1, 0])]; - tensor var_312_end_0 = const()[name = tensor("op_312_end_0"), val = tensor([1, 16, 256])]; - tensor var_312_end_mask_0 = const()[name = tensor("op_312_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_81")]; - tensor var_312 = slice_by_index(begin = var_312_begin_0, end = var_312_end_0, end_mask = var_312_end_mask_0, x = conv_out_1)[name = tensor("op_312")]; - tensor var_315_begin_0 = const()[name = tensor("op_315_begin_0"), val = tensor([0, 1, 0])]; - tensor var_315_end_0 = const()[name = tensor("op_315_end_0"), val = tensor([1, 2, 256])]; - tensor var_315_end_mask_0 = const()[name = tensor("op_315_end_mask_0"), val = tensor([true, false, true])]; - tensor var_315 = slice_by_index(begin = var_315_begin_0, end = var_315_end_0, end_mask = var_315_end_mask_0, x = x_3)[name = tensor("op_315")]; - tensor var_318_begin_0 = const()[name = tensor("op_318_begin_0"), val = tensor([0, 1, 0])]; - tensor var_318_end_0 = const()[name = tensor("op_318_end_0"), val = tensor([1, 16, 256])]; - tensor var_318_end_mask_0 = const()[name = tensor("op_318_end_mask_0"), val = tensor([true, true, true])]; - tensor var_318 = slice_by_index(begin = var_318_begin_0, end = var_318_end_0, end_mask = var_318_end_mask_0, x = input_21)[name = tensor("op_318")]; - tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_34, interleave = rolled_3_interleave_0, values = (var_318, var_315))[name = tensor("rolled_3")]; - tensor clip_2 = clip(alpha = var_22, beta = var_21, x = clip_1)[name = tensor("clip_2")]; - tensor var_323 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_323")]; - tensor var_324_dtype_0 = const()[name = tensor("op_324_dtype_0"), val = tensor("fp32")]; - tensor var_325 = const()[name = tensor("op_325"), val = tensor([1, 16, 1])]; - tensor var_324 = cast(dtype = var_324_dtype_0, x = var_323)[name = tensor("cast_189")]; - tensor fill_mask_3 = reshape(shape = var_325, x = var_324)[name = tensor("fill_mask_3")]; - tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_315)[name = tensor("x_expanded_3")]; - tensor var_329 = sub(x = var_18, y = fill_mask_3)[name = tensor("op_329")]; - tensor var_330 = mul(x = input_21, y = var_329)[name = tensor("op_330")]; - tensor var_331 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_331")]; - tensor filled_window_3 = add(x = var_330, y = var_331)[name = tensor("filled_window_3")]; - tensor var_333 = greater_equal(x = clip_1, y = var_17)[name = tensor("op_333")]; - tensor var_334_dtype_0 = const()[name = tensor("op_334_dtype_0"), val = tensor("fp32")]; - tensor var_335 = const()[name = tensor("op_335"), val = tensor([1, 1, 1])]; - tensor var_334 = cast(dtype = var_334_dtype_0, x = var_333)[name = tensor("cast_188")]; - tensor use_roll_3 = reshape(shape = var_335, x = var_334)[name = tensor("use_roll_3")]; - tensor var_337 = sub(x = var_18, y = use_roll_3)[name = tensor("op_337")]; - tensor var_338 = mul(x = filled_window_3, y = var_337)[name = tensor("op_338")]; - tensor var_339 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_339")]; - tensor input_31 = add(x = var_338, y = var_339)[name = tensor("input_31")]; - tensor var_341 = const()[name = tensor("op_341"), val = tensor(0x1p+0)]; - tensor var_342 = add(x = clip_1, y = var_341)[name = tensor("op_342")]; - tensor clip_3 = clip(alpha = var_22, beta = var_17, x = var_342)[name = tensor("clip_3")]; + tensor var_434_begin_0 = const()[name = tensor("op_434_begin_0"), val = tensor([0, -1, 0])]; + tensor var_434_end_0 = const()[name = tensor("op_434_end_0"), val = tensor([5, 16, 256])]; + tensor var_434_end_mask_0 = const()[name = tensor("op_434_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_434 = slice_by_index(begin = var_434_begin_0, end = var_434_end_0, end_mask = var_434_end_mask_0, x = conv_out_1)[name = tensor("op_434")]; + tensor var_436_perm_0 = const()[name = tensor("op_436_perm_0"), val = tensor([1, 0, 2])]; + tensor var_436 = transpose(perm = var_436_perm_0, x = var_434)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_436)[name = tensor("input_31")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; + tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; + tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; + tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; + tensor var_459 = const()[name = tensor("op_459"), val = tensor(0x1p-1)]; + tensor var_460 = mul(x = input_39, y = var_459)[name = tensor("op_460")]; + tensor input_41 = add(x = var_460, y = input_31)[name = tensor("input_41")]; + tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; + tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_36, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; + tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; + tensor input_45 = layer_norm(axes = input_45_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_43)[name = tensor("input_45")]; + tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; + tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; + tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; + tensor var_489 = const()[name = tensor("op_489"), val = tensor(0x1p-1)]; + tensor var_490 = mul(x = input_51, y = var_489)[name = tensor("op_490")]; + tensor input_53 = add(x = var_490, y = input_43)[name = tensor("input_53")]; tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; - tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_31)[name = tensor("x_7")]; - tensor input_33_perm_0 = const()[name = tensor("input_33_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_9_pad_type_0 = const()[name = tensor("inputs_9_pad_type_0"), val = tensor("valid")]; - tensor inputs_9_strides_0 = const()[name = tensor("inputs_9_strides_0"), val = tensor([1])]; - tensor inputs_9_pad_0 = const()[name = tensor("inputs_9_pad_0"), val = tensor([0, 0])]; - tensor inputs_9_dilations_0 = const()[name = tensor("inputs_9_dilations_0"), val = tensor([1])]; - tensor inputs_9_groups_0 = const()[name = tensor("inputs_9_groups_0"), val = tensor(1)]; - tensor input_33 = transpose(perm = input_33_perm_0, x = x_7)[name = tensor("transpose_80")]; - tensor inputs_9 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_9_dilations_0, groups = inputs_9_groups_0, pad = inputs_9_pad_0, pad_type = inputs_9_pad_type_0, strides = inputs_9_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_33)[name = tensor("inputs_9")]; - tensor var_363_split_sizes_0 = const()[name = tensor("op_363_split_sizes_0"), val = tensor([256, 256])]; - tensor var_363_axis_0 = const()[name = tensor("op_363_axis_0"), val = tensor(1)]; - tensor var_363_0, tensor var_363_1 = split(axis = var_363_axis_0, split_sizes = var_363_split_sizes_0, x = inputs_9)[name = tensor("op_363")]; - tensor var_365 = sigmoid(x = var_363_1)[name = tensor("op_365")]; - tensor inputs_11 = mul(x = var_363_0, y = var_365)[name = tensor("inputs_11")]; - tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; - tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; - tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; - tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_11)[name = tensor("outputs_aug_3")]; - tensor input_35_begin_0 = const()[name = tensor("input_35_begin_0"), val = tensor([0, 0, 0])]; - tensor input_35_end_0 = const()[name = tensor("input_35_end_0"), val = tensor([1, 256, 16])]; - tensor input_35_end_mask_0 = const()[name = tensor("input_35_end_mask_0"), val = tensor([true, true, false])]; - tensor input_35 = slice_by_index(begin = input_35_begin_0, end = input_35_end_0, end_mask = input_35_end_mask_0, x = outputs_aug_3)[name = tensor("input_35")]; - tensor inputs_13 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_35)[name = tensor("inputs_13")]; - tensor input_37 = silu(x = inputs_13)[name = tensor("input_37")]; - tensor input_39_pad_type_0 = const()[name = tensor("input_39_pad_type_0"), val = tensor("valid")]; - tensor input_39_strides_0 = const()[name = tensor("input_39_strides_0"), val = tensor([1])]; - tensor input_39_pad_0 = const()[name = tensor("input_39_pad_0"), val = tensor([0, 0])]; - tensor input_39_dilations_0 = const()[name = tensor("input_39_dilations_0"), val = tensor([1])]; - tensor input_39_groups_0 = const()[name = tensor("input_39_groups_0"), val = tensor(1)]; - tensor input_39 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_37)[name = tensor("input_39")]; - tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_396_begin_0 = const()[name = tensor("op_396_begin_0"), val = tensor([0, -1, 0])]; - tensor var_396_end_0 = const()[name = tensor("op_396_end_0"), val = tensor([1, 16, 256])]; - tensor var_396_end_mask_0 = const()[name = tensor("op_396_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_39)[name = tensor("transpose_79")]; - tensor var_396 = slice_by_index(begin = var_396_begin_0, end = var_396_end_0, end_mask = var_396_end_mask_0, x = conv_out_3)[name = tensor("op_396")]; - tensor var_399_begin_0 = const()[name = tensor("op_399_begin_0"), val = tensor([0, 2, 0])]; - tensor var_399_end_0 = const()[name = tensor("op_399_end_0"), val = tensor([1, 3, 256])]; - tensor var_399_end_mask_0 = const()[name = tensor("op_399_end_mask_0"), val = tensor([true, false, true])]; - tensor var_399 = slice_by_index(begin = var_399_begin_0, end = var_399_end_0, end_mask = var_399_end_mask_0, x = x_3)[name = tensor("op_399")]; - tensor var_402_begin_0 = const()[name = tensor("op_402_begin_0"), val = tensor([0, 1, 0])]; - tensor var_402_end_0 = const()[name = tensor("op_402_end_0"), val = tensor([1, 16, 256])]; - tensor var_402_end_mask_0 = const()[name = tensor("op_402_end_mask_0"), val = tensor([true, true, true])]; - tensor var_402 = slice_by_index(begin = var_402_begin_0, end = var_402_end_0, end_mask = var_402_end_mask_0, x = input_31)[name = tensor("op_402")]; - tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_34, interleave = rolled_5_interleave_0, values = (var_402, var_399))[name = tensor("rolled_5")]; - tensor clip_4 = clip(alpha = var_22, beta = var_21, x = clip_3)[name = tensor("clip_4")]; - tensor var_407 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_407")]; - tensor var_408_dtype_0 = const()[name = tensor("op_408_dtype_0"), val = tensor("fp32")]; - tensor var_409 = const()[name = tensor("op_409"), val = tensor([1, 16, 1])]; - tensor var_408 = cast(dtype = var_408_dtype_0, x = var_407)[name = tensor("cast_187")]; - tensor fill_mask_5 = reshape(shape = var_409, x = var_408)[name = tensor("fill_mask_5")]; - tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_399)[name = tensor("x_expanded_5")]; - tensor var_413 = sub(x = var_18, y = fill_mask_5)[name = tensor("op_413")]; - tensor var_414 = mul(x = input_31, y = var_413)[name = tensor("op_414")]; - tensor var_415 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_415")]; - tensor filled_window_5 = add(x = var_414, y = var_415)[name = tensor("filled_window_5")]; - tensor var_417 = greater_equal(x = clip_3, y = var_17)[name = tensor("op_417")]; - tensor var_418_dtype_0 = const()[name = tensor("op_418_dtype_0"), val = tensor("fp32")]; - tensor var_419 = const()[name = tensor("op_419"), val = tensor([1, 1, 1])]; - tensor var_418 = cast(dtype = var_418_dtype_0, x = var_417)[name = tensor("cast_186")]; - tensor use_roll_5 = reshape(shape = var_419, x = var_418)[name = tensor("use_roll_5")]; - tensor var_421 = sub(x = var_18, y = use_roll_5)[name = tensor("op_421")]; - tensor var_422 = mul(x = filled_window_5, y = var_421)[name = tensor("op_422")]; - tensor var_423 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_423")]; - tensor input_41 = add(x = var_422, y = var_423)[name = tensor("input_41")]; - tensor var_425 = const()[name = tensor("op_425"), val = tensor(0x1p+0)]; - tensor var_426 = add(x = clip_3, y = var_425)[name = tensor("op_426")]; - tensor clip_5 = clip(alpha = var_22, beta = var_17, x = var_426)[name = tensor("clip_5")]; - tensor x_9_axes_0 = const()[name = tensor("x_9_axes_0"), val = tensor([-1])]; - tensor x_9 = layer_norm(axes = x_9_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_41)[name = tensor("x_9")]; - tensor input_43_perm_0 = const()[name = tensor("input_43_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_15_pad_type_0 = const()[name = tensor("inputs_15_pad_type_0"), val = tensor("valid")]; - tensor inputs_15_strides_0 = const()[name = tensor("inputs_15_strides_0"), val = tensor([1])]; - tensor inputs_15_pad_0 = const()[name = tensor("inputs_15_pad_0"), val = tensor([0, 0])]; - tensor inputs_15_dilations_0 = const()[name = tensor("inputs_15_dilations_0"), val = tensor([1])]; - tensor inputs_15_groups_0 = const()[name = tensor("inputs_15_groups_0"), val = tensor(1)]; - tensor input_43 = transpose(perm = input_43_perm_0, x = x_9)[name = tensor("transpose_78")]; - tensor inputs_15 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_15_dilations_0, groups = inputs_15_groups_0, pad = inputs_15_pad_0, pad_type = inputs_15_pad_type_0, strides = inputs_15_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_43)[name = tensor("inputs_15")]; - tensor var_447_split_sizes_0 = const()[name = tensor("op_447_split_sizes_0"), val = tensor([256, 256])]; - tensor var_447_axis_0 = const()[name = tensor("op_447_axis_0"), val = tensor(1)]; - tensor var_447_0, tensor var_447_1 = split(axis = var_447_axis_0, split_sizes = var_447_split_sizes_0, x = inputs_15)[name = tensor("op_447")]; - tensor var_449 = sigmoid(x = var_447_1)[name = tensor("op_449")]; - tensor inputs_17 = mul(x = var_447_0, y = var_449)[name = tensor("inputs_17")]; - tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; - tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; - tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; - tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_17)[name = tensor("outputs_aug_5")]; - tensor input_45_begin_0 = const()[name = tensor("input_45_begin_0"), val = tensor([0, 0, 0])]; - tensor input_45_end_0 = const()[name = tensor("input_45_end_0"), val = tensor([1, 256, 16])]; - tensor input_45_end_mask_0 = const()[name = tensor("input_45_end_mask_0"), val = tensor([true, true, false])]; - tensor input_45 = slice_by_index(begin = input_45_begin_0, end = input_45_end_0, end_mask = input_45_end_mask_0, x = outputs_aug_5)[name = tensor("input_45")]; - tensor inputs_19 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_45)[name = tensor("inputs_19")]; - tensor input_47 = silu(x = inputs_19)[name = tensor("input_47")]; - tensor input_49_pad_type_0 = const()[name = tensor("input_49_pad_type_0"), val = tensor("valid")]; - tensor input_49_strides_0 = const()[name = tensor("input_49_strides_0"), val = tensor([1])]; - tensor input_49_pad_0 = const()[name = tensor("input_49_pad_0"), val = tensor([0, 0])]; - tensor input_49_dilations_0 = const()[name = tensor("input_49_dilations_0"), val = tensor([1])]; - tensor input_49_groups_0 = const()[name = tensor("input_49_groups_0"), val = tensor(1)]; - tensor input_49 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_49_dilations_0, groups = input_49_groups_0, pad = input_49_pad_0, pad_type = input_49_pad_type_0, strides = input_49_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_47)[name = tensor("input_49")]; - tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_480_begin_0 = const()[name = tensor("op_480_begin_0"), val = tensor([0, -1, 0])]; - tensor var_480_end_0 = const()[name = tensor("op_480_end_0"), val = tensor([1, 16, 256])]; - tensor var_480_end_mask_0 = const()[name = tensor("op_480_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_49)[name = tensor("transpose_77")]; - tensor var_480 = slice_by_index(begin = var_480_begin_0, end = var_480_end_0, end_mask = var_480_end_mask_0, x = conv_out_5)[name = tensor("op_480")]; - tensor var_483_begin_0 = const()[name = tensor("op_483_begin_0"), val = tensor([0, 3, 0])]; - tensor var_483_end_0 = const()[name = tensor("op_483_end_0"), val = tensor([1, 4, 256])]; - tensor var_483_end_mask_0 = const()[name = tensor("op_483_end_mask_0"), val = tensor([true, false, true])]; - tensor var_483 = slice_by_index(begin = var_483_begin_0, end = var_483_end_0, end_mask = var_483_end_mask_0, x = x_3)[name = tensor("op_483")]; - tensor var_486_begin_0 = const()[name = tensor("op_486_begin_0"), val = tensor([0, 1, 0])]; - tensor var_486_end_0 = const()[name = tensor("op_486_end_0"), val = tensor([1, 16, 256])]; - tensor var_486_end_mask_0 = const()[name = tensor("op_486_end_mask_0"), val = tensor([true, true, true])]; - tensor var_486 = slice_by_index(begin = var_486_begin_0, end = var_486_end_0, end_mask = var_486_end_mask_0, x = input_41)[name = tensor("op_486")]; - tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; - tensor rolled_7 = concat(axis = var_34, interleave = rolled_7_interleave_0, values = (var_486, var_483))[name = tensor("rolled_7")]; - tensor clip_6 = clip(alpha = var_22, beta = var_21, x = clip_5)[name = tensor("clip_6")]; - tensor var_491 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_491")]; - tensor var_492_dtype_0 = const()[name = tensor("op_492_dtype_0"), val = tensor("fp32")]; - tensor var_493 = const()[name = tensor("op_493"), val = tensor([1, 16, 1])]; - tensor var_492 = cast(dtype = var_492_dtype_0, x = var_491)[name = tensor("cast_185")]; - tensor fill_mask_7 = reshape(shape = var_493, x = var_492)[name = tensor("fill_mask_7")]; - tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_483)[name = tensor("x_expanded_7")]; - tensor var_497 = sub(x = var_18, y = fill_mask_7)[name = tensor("op_497")]; - tensor var_498 = mul(x = input_41, y = var_497)[name = tensor("op_498")]; - tensor var_499 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_499")]; - tensor filled_window_7 = add(x = var_498, y = var_499)[name = tensor("filled_window_7")]; - tensor var_501 = greater_equal(x = clip_5, y = var_17)[name = tensor("op_501")]; - tensor var_502_dtype_0 = const()[name = tensor("op_502_dtype_0"), val = tensor("fp32")]; - tensor var_503 = const()[name = tensor("op_503"), val = tensor([1, 1, 1])]; - tensor var_502 = cast(dtype = var_502_dtype_0, x = var_501)[name = tensor("cast_184")]; - tensor use_roll_7 = reshape(shape = var_503, x = var_502)[name = tensor("use_roll_7")]; - tensor var_505 = sub(x = var_18, y = use_roll_7)[name = tensor("op_505")]; - tensor var_506 = mul(x = filled_window_7, y = var_505)[name = tensor("op_506")]; - tensor var_507 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_507")]; - tensor input_51 = add(x = var_506, y = var_507)[name = tensor("input_51")]; - tensor var_509 = const()[name = tensor("op_509"), val = tensor(0x1p+0)]; - tensor var_510 = add(x = clip_5, y = var_509)[name = tensor("op_510")]; - tensor clip_7 = clip(alpha = var_22, beta = var_17, x = var_510)[name = tensor("clip_7")]; - tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; - tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_51)[name = tensor("x_11")]; - tensor input_53_perm_0 = const()[name = tensor("input_53_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_21_pad_type_0 = const()[name = tensor("inputs_21_pad_type_0"), val = tensor("valid")]; - tensor inputs_21_strides_0 = const()[name = tensor("inputs_21_strides_0"), val = tensor([1])]; - tensor inputs_21_pad_0 = const()[name = tensor("inputs_21_pad_0"), val = tensor([0, 0])]; - tensor inputs_21_dilations_0 = const()[name = tensor("inputs_21_dilations_0"), val = tensor([1])]; - tensor inputs_21_groups_0 = const()[name = tensor("inputs_21_groups_0"), val = tensor(1)]; - tensor input_53 = transpose(perm = input_53_perm_0, x = x_11)[name = tensor("transpose_76")]; - tensor inputs_21 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_21_dilations_0, groups = inputs_21_groups_0, pad = inputs_21_pad_0, pad_type = inputs_21_pad_type_0, strides = inputs_21_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_53)[name = tensor("inputs_21")]; - tensor var_531_split_sizes_0 = const()[name = tensor("op_531_split_sizes_0"), val = tensor([256, 256])]; - tensor var_531_axis_0 = const()[name = tensor("op_531_axis_0"), val = tensor(1)]; - tensor var_531_0, tensor var_531_1 = split(axis = var_531_axis_0, split_sizes = var_531_split_sizes_0, x = inputs_21)[name = tensor("op_531")]; - tensor var_533 = sigmoid(x = var_531_1)[name = tensor("op_533")]; - tensor inputs_23 = mul(x = var_531_0, y = var_533)[name = tensor("inputs_23")]; - tensor outputs_aug_7_pad_type_0 = const()[name = tensor("outputs_aug_7_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_7_pad_0 = const()[name = tensor("outputs_aug_7_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_7_groups_0 = const()[name = tensor("outputs_aug_7_groups_0"), val = tensor(256)]; - tensor outputs_aug_7_strides_0 = const()[name = tensor("outputs_aug_7_strides_0"), val = tensor([1])]; - tensor outputs_aug_7_dilations_0 = const()[name = tensor("outputs_aug_7_dilations_0"), val = tensor([1])]; - tensor outputs_aug_7 = conv(dilations = outputs_aug_7_dilations_0, groups = outputs_aug_7_groups_0, pad = outputs_aug_7_pad_0, pad_type = outputs_aug_7_pad_type_0, strides = outputs_aug_7_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_23)[name = tensor("outputs_aug_7")]; - tensor input_55_begin_0 = const()[name = tensor("input_55_begin_0"), val = tensor([0, 0, 0])]; - tensor input_55_end_0 = const()[name = tensor("input_55_end_0"), val = tensor([1, 256, 16])]; - tensor input_55_end_mask_0 = const()[name = tensor("input_55_end_mask_0"), val = tensor([true, true, false])]; - tensor input_55 = slice_by_index(begin = input_55_begin_0, end = input_55_end_0, end_mask = input_55_end_mask_0, x = outputs_aug_7)[name = tensor("input_55")]; - tensor inputs_25 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_55)[name = tensor("inputs_25")]; - tensor input_57 = silu(x = inputs_25)[name = tensor("input_57")]; - tensor input_59_pad_type_0 = const()[name = tensor("input_59_pad_type_0"), val = tensor("valid")]; - tensor input_59_strides_0 = const()[name = tensor("input_59_strides_0"), val = tensor([1])]; - tensor input_59_pad_0 = const()[name = tensor("input_59_pad_0"), val = tensor([0, 0])]; - tensor input_59_dilations_0 = const()[name = tensor("input_59_dilations_0"), val = tensor([1])]; - tensor input_59_groups_0 = const()[name = tensor("input_59_groups_0"), val = tensor(1)]; - tensor input_59 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_59_dilations_0, groups = input_59_groups_0, pad = input_59_pad_0, pad_type = input_59_pad_type_0, strides = input_59_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_57)[name = tensor("input_59")]; - tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_564_begin_0 = const()[name = tensor("op_564_begin_0"), val = tensor([0, -1, 0])]; - tensor var_564_end_0 = const()[name = tensor("op_564_end_0"), val = tensor([1, 16, 256])]; - tensor var_564_end_mask_0 = const()[name = tensor("op_564_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_59)[name = tensor("transpose_75")]; - tensor var_564 = slice_by_index(begin = var_564_begin_0, end = var_564_end_0, end_mask = var_564_end_mask_0, x = conv_out_7)[name = tensor("op_564")]; - tensor var_567_begin_0 = const()[name = tensor("op_567_begin_0"), val = tensor([0, 4, 0])]; - tensor var_567_end_0 = const()[name = tensor("op_567_end_0"), val = tensor([1, 1, 256])]; - tensor var_567_end_mask_0 = const()[name = tensor("op_567_end_mask_0"), val = tensor([true, true, true])]; - tensor var_567 = slice_by_index(begin = var_567_begin_0, end = var_567_end_0, end_mask = var_567_end_mask_0, x = x_3)[name = tensor("op_567")]; - tensor var_570_begin_0 = const()[name = tensor("op_570_begin_0"), val = tensor([0, 1, 0])]; - tensor var_570_end_0 = const()[name = tensor("op_570_end_0"), val = tensor([1, 16, 256])]; - tensor var_570_end_mask_0 = const()[name = tensor("op_570_end_mask_0"), val = tensor([true, true, true])]; - tensor var_570 = slice_by_index(begin = var_570_begin_0, end = var_570_end_0, end_mask = var_570_end_mask_0, x = input_51)[name = tensor("op_570")]; - tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; - tensor rolled_9 = concat(axis = var_34, interleave = rolled_9_interleave_0, values = (var_570, var_567))[name = tensor("rolled_9")]; - tensor clip_8 = clip(alpha = var_22, beta = var_21, x = clip_7)[name = tensor("clip_8")]; - tensor var_575 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_575")]; - tensor var_576_dtype_0 = const()[name = tensor("op_576_dtype_0"), val = tensor("fp32")]; - tensor var_577 = const()[name = tensor("op_577"), val = tensor([1, 16, 1])]; - tensor var_576 = cast(dtype = var_576_dtype_0, x = var_575)[name = tensor("cast_183")]; - tensor fill_mask_9 = reshape(shape = var_577, x = var_576)[name = tensor("fill_mask_9")]; - tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_567)[name = tensor("x_expanded_9")]; - tensor var_581 = sub(x = var_18, y = fill_mask_9)[name = tensor("op_581")]; - tensor var_582 = mul(x = input_51, y = var_581)[name = tensor("op_582")]; - tensor var_583 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_583")]; - tensor filled_window_9 = add(x = var_582, y = var_583)[name = tensor("filled_window_9")]; - tensor var_585 = greater_equal(x = clip_7, y = var_17)[name = tensor("op_585")]; - tensor var_586_dtype_0 = const()[name = tensor("op_586_dtype_0"), val = tensor("fp32")]; - tensor var_587 = const()[name = tensor("op_587"), val = tensor([1, 1, 1])]; - tensor var_586 = cast(dtype = var_586_dtype_0, x = var_585)[name = tensor("cast_182")]; - tensor use_roll_9 = reshape(shape = var_587, x = var_586)[name = tensor("use_roll_9")]; - tensor var_589 = sub(x = var_18, y = use_roll_9)[name = tensor("op_589")]; - tensor var_590 = mul(x = filled_window_9, y = var_589)[name = tensor("op_590")]; - tensor var_591 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_591")]; - tensor input_61 = add(x = var_590, y = var_591)[name = tensor("input_61")]; - tensor var_593 = const()[name = tensor("op_593"), val = tensor(0x1p+0)]; - tensor var_594 = add(x = clip_7, y = var_593)[name = tensor("op_594")]; - tensor clip_9 = clip(alpha = var_22, beta = var_17, x = var_594)[name = tensor("clip_9")]; - tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; - tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_61)[name = tensor("x_13")]; - tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_27_pad_type_0 = const()[name = tensor("inputs_27_pad_type_0"), val = tensor("valid")]; - tensor inputs_27_strides_0 = const()[name = tensor("inputs_27_strides_0"), val = tensor([1])]; - tensor inputs_27_pad_0 = const()[name = tensor("inputs_27_pad_0"), val = tensor([0, 0])]; - tensor inputs_27_dilations_0 = const()[name = tensor("inputs_27_dilations_0"), val = tensor([1])]; - tensor inputs_27_groups_0 = const()[name = tensor("inputs_27_groups_0"), val = tensor(1)]; - tensor input_63 = transpose(perm = input_63_perm_0, x = x_13)[name = tensor("transpose_74")]; - tensor inputs_27 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_27_dilations_0, groups = inputs_27_groups_0, pad = inputs_27_pad_0, pad_type = inputs_27_pad_type_0, strides = inputs_27_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_27")]; - tensor var_615_split_sizes_0 = const()[name = tensor("op_615_split_sizes_0"), val = tensor([256, 256])]; - tensor var_615_axis_0 = const()[name = tensor("op_615_axis_0"), val = tensor(1)]; - tensor var_615_0, tensor var_615_1 = split(axis = var_615_axis_0, split_sizes = var_615_split_sizes_0, x = inputs_27)[name = tensor("op_615")]; - tensor var_617 = sigmoid(x = var_615_1)[name = tensor("op_617")]; - tensor inputs_29 = mul(x = var_615_0, y = var_617)[name = tensor("inputs_29")]; - tensor outputs_aug_9_pad_type_0 = const()[name = tensor("outputs_aug_9_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_9_pad_0 = const()[name = tensor("outputs_aug_9_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_9_groups_0 = const()[name = tensor("outputs_aug_9_groups_0"), val = tensor(256)]; - tensor outputs_aug_9_strides_0 = const()[name = tensor("outputs_aug_9_strides_0"), val = tensor([1])]; - tensor outputs_aug_9_dilations_0 = const()[name = tensor("outputs_aug_9_dilations_0"), val = tensor([1])]; - tensor outputs_aug_9 = conv(dilations = outputs_aug_9_dilations_0, groups = outputs_aug_9_groups_0, pad = outputs_aug_9_pad_0, pad_type = outputs_aug_9_pad_type_0, strides = outputs_aug_9_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_29)[name = tensor("outputs_aug_9")]; - tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; - tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([1, 256, 16])]; - tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; - tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_9)[name = tensor("input_65")]; - tensor inputs_31 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_65)[name = tensor("inputs_31")]; - tensor input_67 = silu(x = inputs_31)[name = tensor("input_67")]; - tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; - tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; - tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; - tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; - tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; - tensor input_69 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; - tensor conv_out_9_perm_0 = const()[name = tensor("conv_out_9_perm_0"), val = tensor([0, 2, 1])]; - tensor var_648_begin_0 = const()[name = tensor("op_648_begin_0"), val = tensor([0, -1, 0])]; - tensor var_648_end_0 = const()[name = tensor("op_648_end_0"), val = tensor([1, 16, 256])]; - tensor var_648_end_mask_0 = const()[name = tensor("op_648_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_9 = transpose(perm = conv_out_9_perm_0, x = input_69)[name = tensor("transpose_73")]; - tensor var_648 = slice_by_index(begin = var_648_begin_0, end = var_648_end_0, end_mask = var_648_end_mask_0, x = conv_out_9)[name = tensor("op_648")]; - tensor var_651_interleave_0 = const()[name = tensor("op_651_interleave_0"), val = tensor(false)]; - tensor var_651 = concat(axis = var_34, interleave = var_651_interleave_0, values = (var_312, var_396, var_480, var_564, var_648))[name = tensor("op_651")]; - tensor input_71 = add(x = x_3, y = var_651)[name = tensor("input_71")]; - tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; - tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; - tensor inputs_33 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_8")]; - tensor input_75 = silu(x = inputs_33)[name = tensor("input_75")]; - tensor input_79 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_9")]; - tensor var_674 = const()[name = tensor("op_674"), val = tensor(0x1p-1)]; - tensor var_675 = mul(x = input_79, y = var_674)[name = tensor("op_675")]; - tensor input_81 = add(x = var_675, y = input_71)[name = tensor("input_81")]; - tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; - tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_36, gamma = encoder_layer_norm_0_weight, x = input_81)[name = tensor("input_83")]; - tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; - tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; - tensor inputs_35 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_10")]; - tensor input_87 = silu(x = inputs_35)[name = tensor("input_87")]; - tensor input_91 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_11")]; - tensor var_704 = const()[name = tensor("op_704"), val = tensor(0x1p-1)]; - tensor var_705 = mul(x = input_91, y = var_704)[name = tensor("op_705")]; - tensor input_93 = add(x = var_705, y = input_83)[name = tensor("input_93")]; - tensor x_15_axes_0 = const()[name = tensor("x_15_axes_0"), val = tensor([-1])]; - tensor x_15 = layer_norm(axes = x_15_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_36, gamma = encoder_ret_lns_1_weight, x = input_93)[name = tensor("x_15")]; + tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_36, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_3_end_0 = const()[name = tensor("prev_kv_3_end_0"), val = tensor([2, 1, 4, 64, 64])]; tensor prev_kv_3_end_mask_0 = const()[name = tensor("prev_kv_3_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -709,462 +556,308 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_719 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_15)[name = tensor("linear_12")]; - tensor var_720 = const()[name = tensor("op_720"), val = tensor([1, 5, 4, 64])]; - tensor var_721 = reshape(shape = var_720, x = var_719)[name = tensor("op_721")]; + tensor var_504 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_505 = const()[name = tensor("op_505"), val = tensor([1, 5, 4, 64])]; + tensor var_506 = reshape(shape = var_505, x = var_504)[name = tensor("op_506")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_725 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_15)[name = tensor("linear_13")]; - tensor var_726 = const()[name = tensor("op_726"), val = tensor(0x1p-3)]; - tensor var_727 = mul(x = var_725, y = var_726)[name = tensor("op_727")]; - tensor var_728 = const()[name = tensor("op_728"), val = tensor([1, 5, 4, 64])]; - tensor var_729 = reshape(shape = var_728, x = var_727)[name = tensor("op_729")]; + tensor var_510 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_511 = const()[name = tensor("op_511"), val = tensor(0x1p-3)]; + tensor var_512 = mul(x = var_510, y = var_511)[name = tensor("op_512")]; + tensor var_513 = const()[name = tensor("op_513"), val = tensor([1, 5, 4, 64])]; + tensor var_514 = reshape(shape = var_513, x = var_512)[name = tensor("op_514")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_733 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_15)[name = tensor("linear_14")]; - tensor var_734 = const()[name = tensor("op_734"), val = tensor([1, 5, 4, 64])]; - tensor var_735 = reshape(shape = var_734, x = var_733)[name = tensor("op_735")]; + tensor var_518 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_519 = const()[name = tensor("op_519"), val = tensor([1, 5, 4, 64])]; + tensor var_520 = reshape(shape = var_519, x = var_518)[name = tensor("op_520")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_97 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_15)[name = tensor("linear_15")]; + tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; tensor s_t_3 = add(x = prev_scale_3, y = encoder__t_index)[name = tensor("s_t_3")]; tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_729)[name = tensor("transpose_71")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_721)[name = tensor("transpose_72")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_514)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_506)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_745 = const()[name = tensor("op_745"), val = tensor([5, 1])]; - tensor var_746 = reshape(shape = var_745, x = sqrt_s_t_3)[name = tensor("op_746")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_746)[name = tensor("M_3")]; - tensor var_748 = mul(x = qk_3, y = M_3)[name = tensor("op_748")]; + tensor var_530 = const()[name = tensor("op_530"), val = tensor([5, 1])]; + tensor var_531 = reshape(shape = var_530, x = sqrt_s_t_3)[name = tensor("op_531")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_531)[name = tensor("M_3")]; + tensor var_533 = mul(x = qk_3, y = M_3)[name = tensor("op_533")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_735)[name = tensor("transpose_70")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_748, y = v_3)[name = tensor("inner_3")]; - tensor var_750_transpose_x_0 = const()[name = tensor("op_750_transpose_x_0"), val = tensor(false)]; - tensor var_750_transpose_y_0 = const()[name = tensor("op_750_transpose_y_0"), val = tensor(false)]; - tensor var_750 = matmul(transpose_x = var_750_transpose_x_0, transpose_y = var_750_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_750")]; - tensor var_751 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_751")]; - tensor var_752 = const()[name = tensor("op_752"), val = tensor([1, 1, 5, 1])]; - tensor var_753 = reshape(shape = var_752, x = var_751)[name = tensor("op_753")]; - tensor cross_3 = mul(x = var_750, y = var_753)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_520)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_533, y = v_3)[name = tensor("inner_3")]; + tensor var_535_transpose_x_0 = const()[name = tensor("op_535_transpose_x_0"), val = tensor(false)]; + tensor var_535_transpose_y_0 = const()[name = tensor("op_535_transpose_y_0"), val = tensor(false)]; + tensor var_535 = matmul(transpose_x = var_535_transpose_x_0, transpose_y = var_535_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_535")]; + tensor var_536 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_536")]; + tensor var_537 = const()[name = tensor("op_537"), val = tensor([1, 1, 5, 1])]; + tensor var_538 = reshape(shape = var_537, x = var_536)[name = tensor("op_538")]; + tensor cross_3 = mul(x = var_535, y = var_538)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_756 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_756")]; - tensor var_758_transpose_x_1 = const()[name = tensor("op_758_transpose_x_1"), val = tensor(true)]; - tensor var_758_transpose_y_1 = const()[name = tensor("op_758_transpose_y_1"), val = tensor(false)]; - tensor var_758 = matmul(transpose_x = var_758_transpose_x_1, transpose_y = var_758_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_758")]; - tensor new_kv_unnorm_3 = add(x = var_756, y = var_758)[name = tensor("new_kv_unnorm_3")]; - tensor var_760 = const()[name = tensor("op_760"), val = tensor(0x1.4p+2)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_760)[name = tensor("new_scale_3")]; - tensor var_762 = sqrt(x = new_scale_3)[name = tensor("op_762")]; - tensor var_763 = real_div(x = new_kv_unnorm_3, y = var_762)[name = tensor("op_763")]; - tensor var_764_perm_0 = const()[name = tensor("op_764_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_541 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_541")]; + tensor var_543_transpose_x_1 = const()[name = tensor("op_543_transpose_x_1"), val = tensor(true)]; + tensor var_543_transpose_y_1 = const()[name = tensor("op_543_transpose_y_1"), val = tensor(false)]; + tensor var_543 = matmul(transpose_x = var_543_transpose_x_1, transpose_y = var_543_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_543")]; + tensor new_kv_unnorm_3 = add(x = var_541, y = var_543)[name = tensor("new_kv_unnorm_3")]; + tensor var_545 = const()[name = tensor("op_545"), val = tensor(0x1.4p+2)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_545)[name = tensor("new_scale_3")]; + tensor var_547 = sqrt(x = new_scale_3)[name = tensor("op_547")]; + tensor var_548 = real_div(x = new_kv_unnorm_3, y = var_547)[name = tensor("op_548")]; + tensor var_549_perm_0 = const()[name = tensor("op_549_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_764 = transpose(perm = var_764_perm_0, x = out_7)[name = tensor("transpose_69")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_25, x = var_764)[name = tensor("out_9")]; - tensor var_768 = const()[name = tensor("op_768"), val = tensor([1, 5, 256])]; - tensor out_11 = reshape(shape = var_768, x = out_9)[name = tensor("out_11")]; - tensor var_770 = silu(x = input_97)[name = tensor("op_770")]; - tensor input_99 = mul(x = var_770, y = out_11)[name = tensor("input_99")]; - tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_99)[name = tensor("linear_16")]; - tensor x_17 = add(x = input_93, y = ret_out_3)[name = tensor("x_17")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor var_549 = transpose(perm = var_549_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_25, x = var_549)[name = tensor("out_9")]; + tensor var_553 = const()[name = tensor("op_553"), val = tensor([1, 5, 256])]; + tensor out_11 = reshape(shape = var_553, x = out_9)[name = tensor("out_11")]; + tensor var_555 = silu(x = input_57)[name = tensor("op_555")]; + tensor input_59 = mul(x = var_555, y = out_11)[name = tensor("input_59")]; + tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; + tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; + tensor window_13_begin_0 = const()[name = tensor("window_13_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_13_end_0 = const()[name = tensor("window_13_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_13_end_mask_0 = const()[name = tensor("window_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_13_squeeze_mask_0 = const()[name = tensor("window_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_13 = slice_by_index(begin = window_13_begin_0, end = window_13_end_0, end_mask = window_13_end_mask_0, squeeze_mask = window_13_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_13")]; tensor filled_11_begin_0 = const()[name = tensor("filled_11_begin_0"), val = tensor([1, 0])]; tensor filled_11_end_0 = const()[name = tensor("filled_11_end_0"), val = tensor([2, 1])]; tensor filled_11_end_mask_0 = const()[name = tensor("filled_11_end_mask_0"), val = tensor([false, true])]; tensor filled_11_squeeze_mask_0 = const()[name = tensor("filled_11_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_11 = slice_by_index(begin = filled_11_begin_0, end = filled_11_end_0, end_mask = filled_11_end_mask_0, squeeze_mask = filled_11_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_11")]; - tensor var_779_begin_0 = const()[name = tensor("op_779_begin_0"), val = tensor([0, 0, 0])]; - tensor var_779_end_0 = const()[name = tensor("op_779_end_0"), val = tensor([1, 1, 256])]; - tensor var_779_end_mask_0 = const()[name = tensor("op_779_end_mask_0"), val = tensor([true, false, true])]; - tensor var_779 = slice_by_index(begin = var_779_begin_0, end = var_779_end_0, end_mask = var_779_end_mask_0, x = x_17)[name = tensor("op_779")]; - tensor var_782_begin_0 = const()[name = tensor("op_782_begin_0"), val = tensor([0, 1, 0])]; - tensor var_782_end_0 = const()[name = tensor("op_782_end_0"), val = tensor([1, 16, 256])]; - tensor var_782_end_mask_0 = const()[name = tensor("op_782_end_mask_0"), val = tensor([true, true, true])]; - tensor var_782 = slice_by_index(begin = var_782_begin_0, end = var_782_end_0, end_mask = var_782_end_mask_0, x = window_3)[name = tensor("op_782")]; + tensor var_564_begin_0 = const()[name = tensor("op_564_begin_0"), val = tensor([0, 0, 0])]; + tensor var_564_end_0 = const()[name = tensor("op_564_end_0"), val = tensor([1, 1, 256])]; + tensor var_564_end_mask_0 = const()[name = tensor("op_564_end_mask_0"), val = tensor([true, false, true])]; + tensor var_564 = slice_by_index(begin = var_564_begin_0, end = var_564_end_0, end_mask = var_564_end_mask_0, x = x_9)[name = tensor("op_564")]; + tensor var_567_begin_0 = const()[name = tensor("op_567_begin_0"), val = tensor([0, 1, 0])]; + tensor var_567_end_0 = const()[name = tensor("op_567_end_0"), val = tensor([1, 16, 256])]; + tensor var_567_end_mask_0 = const()[name = tensor("op_567_end_mask_0"), val = tensor([true, true, true])]; + tensor var_567 = slice_by_index(begin = var_567_begin_0, end = var_567_end_0, end_mask = var_567_end_mask_0, x = window_13)[name = tensor("op_567")]; tensor rolled_11_interleave_0 = const()[name = tensor("rolled_11_interleave_0"), val = tensor(false)]; - tensor rolled_11 = concat(axis = var_34, interleave = rolled_11_interleave_0, values = (var_782, var_779))[name = tensor("rolled_11")]; + tensor rolled_11 = concat(axis = var_34, interleave = rolled_11_interleave_0, values = (var_567, var_564))[name = tensor("rolled_11")]; tensor clip_10 = clip(alpha = var_22, beta = var_21, x = filled_11)[name = tensor("clip_10")]; - tensor var_787 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_787")]; - tensor var_788_dtype_0 = const()[name = tensor("op_788_dtype_0"), val = tensor("fp32")]; - tensor var_789 = const()[name = tensor("op_789"), val = tensor([1, 16, 1])]; - tensor var_788 = cast(dtype = var_788_dtype_0, x = var_787)[name = tensor("cast_181")]; - tensor fill_mask_11 = reshape(shape = var_789, x = var_788)[name = tensor("fill_mask_11")]; + tensor var_572 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_572")]; + tensor var_573_dtype_0 = const()[name = tensor("op_573_dtype_0"), val = tensor("fp32")]; + tensor var_574 = const()[name = tensor("op_574"), val = tensor([1, 16, 1])]; + tensor var_573 = cast(dtype = var_573_dtype_0, x = var_572)[name = tensor("cast_181")]; + tensor fill_mask_11 = reshape(shape = var_574, x = var_573)[name = tensor("fill_mask_11")]; tensor x_expanded_11_reps_0 = const()[name = tensor("x_expanded_11_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_779)[name = tensor("x_expanded_11")]; - tensor var_793 = sub(x = var_18, y = fill_mask_11)[name = tensor("op_793")]; - tensor var_794 = mul(x = window_3, y = var_793)[name = tensor("op_794")]; - tensor var_795 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_795")]; - tensor filled_window_11 = add(x = var_794, y = var_795)[name = tensor("filled_window_11")]; - tensor var_797 = greater_equal(x = filled_11, y = var_17)[name = tensor("op_797")]; - tensor var_798_dtype_0 = const()[name = tensor("op_798_dtype_0"), val = tensor("fp32")]; - tensor var_799 = const()[name = tensor("op_799"), val = tensor([1, 1, 1])]; - tensor var_798 = cast(dtype = var_798_dtype_0, x = var_797)[name = tensor("cast_180")]; - tensor use_roll_11 = reshape(shape = var_799, x = var_798)[name = tensor("use_roll_11")]; - tensor var_801 = sub(x = var_18, y = use_roll_11)[name = tensor("op_801")]; - tensor var_802 = mul(x = filled_window_11, y = var_801)[name = tensor("op_802")]; - tensor var_803 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_803")]; - tensor input_101 = add(x = var_802, y = var_803)[name = tensor("input_101")]; - tensor var_805 = const()[name = tensor("op_805"), val = tensor(0x1p+0)]; - tensor var_806 = add(x = filled_11, y = var_805)[name = tensor("op_806")]; - tensor clip_11 = clip(alpha = var_22, beta = var_17, x = var_806)[name = tensor("clip_11")]; - tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; - tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_101)[name = tensor("x_19")]; - tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_37_pad_type_0 = const()[name = tensor("inputs_37_pad_type_0"), val = tensor("valid")]; - tensor inputs_37_strides_0 = const()[name = tensor("inputs_37_strides_0"), val = tensor([1])]; - tensor inputs_37_pad_0 = const()[name = tensor("inputs_37_pad_0"), val = tensor([0, 0])]; - tensor inputs_37_dilations_0 = const()[name = tensor("inputs_37_dilations_0"), val = tensor([1])]; - tensor inputs_37_groups_0 = const()[name = tensor("inputs_37_groups_0"), val = tensor(1)]; - tensor input_103 = transpose(perm = input_103_perm_0, x = x_19)[name = tensor("transpose_68")]; - tensor inputs_37 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_37_dilations_0, groups = inputs_37_groups_0, pad = inputs_37_pad_0, pad_type = inputs_37_pad_type_0, strides = inputs_37_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_37")]; - tensor var_827_split_sizes_0 = const()[name = tensor("op_827_split_sizes_0"), val = tensor([256, 256])]; - tensor var_827_axis_0 = const()[name = tensor("op_827_axis_0"), val = tensor(1)]; - tensor var_827_0, tensor var_827_1 = split(axis = var_827_axis_0, split_sizes = var_827_split_sizes_0, x = inputs_37)[name = tensor("op_827")]; - tensor var_829 = sigmoid(x = var_827_1)[name = tensor("op_829")]; - tensor inputs_39 = mul(x = var_827_0, y = var_829)[name = tensor("inputs_39")]; - tensor outputs_aug_11_pad_type_0 = const()[name = tensor("outputs_aug_11_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_11_pad_0 = const()[name = tensor("outputs_aug_11_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_11_groups_0 = const()[name = tensor("outputs_aug_11_groups_0"), val = tensor(256)]; - tensor outputs_aug_11_strides_0 = const()[name = tensor("outputs_aug_11_strides_0"), val = tensor([1])]; - tensor outputs_aug_11_dilations_0 = const()[name = tensor("outputs_aug_11_dilations_0"), val = tensor([1])]; - tensor outputs_aug_11 = conv(dilations = outputs_aug_11_dilations_0, groups = outputs_aug_11_groups_0, pad = outputs_aug_11_pad_0, pad_type = outputs_aug_11_pad_type_0, strides = outputs_aug_11_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_39)[name = tensor("outputs_aug_11")]; - tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; - tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([1, 256, 16])]; - tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; - tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_11)[name = tensor("input_105")]; - tensor inputs_41 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_105)[name = tensor("inputs_41")]; - tensor input_107 = silu(x = inputs_41)[name = tensor("input_107")]; - tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; - tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; - tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; - tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; - tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; - tensor input_109 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; - tensor conv_out_11_perm_0 = const()[name = tensor("conv_out_11_perm_0"), val = tensor([0, 2, 1])]; - tensor var_860_begin_0 = const()[name = tensor("op_860_begin_0"), val = tensor([0, -1, 0])]; - tensor var_860_end_0 = const()[name = tensor("op_860_end_0"), val = tensor([1, 16, 256])]; - tensor var_860_end_mask_0 = const()[name = tensor("op_860_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_11 = transpose(perm = conv_out_11_perm_0, x = input_109)[name = tensor("transpose_67")]; - tensor var_860 = slice_by_index(begin = var_860_begin_0, end = var_860_end_0, end_mask = var_860_end_mask_0, x = conv_out_11)[name = tensor("op_860")]; - tensor var_863_begin_0 = const()[name = tensor("op_863_begin_0"), val = tensor([0, 1, 0])]; - tensor var_863_end_0 = const()[name = tensor("op_863_end_0"), val = tensor([1, 2, 256])]; - tensor var_863_end_mask_0 = const()[name = tensor("op_863_end_mask_0"), val = tensor([true, false, true])]; - tensor var_863 = slice_by_index(begin = var_863_begin_0, end = var_863_end_0, end_mask = var_863_end_mask_0, x = x_17)[name = tensor("op_863")]; - tensor var_866_begin_0 = const()[name = tensor("op_866_begin_0"), val = tensor([0, 1, 0])]; - tensor var_866_end_0 = const()[name = tensor("op_866_end_0"), val = tensor([1, 16, 256])]; - tensor var_866_end_mask_0 = const()[name = tensor("op_866_end_mask_0"), val = tensor([true, true, true])]; - tensor var_866 = slice_by_index(begin = var_866_begin_0, end = var_866_end_0, end_mask = var_866_end_mask_0, x = input_101)[name = tensor("op_866")]; + tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_564)[name = tensor("x_expanded_11")]; + tensor var_578 = sub(x = var_18, y = fill_mask_11)[name = tensor("op_578")]; + tensor var_579 = mul(x = window_13, y = var_578)[name = tensor("op_579")]; + tensor var_580 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_580")]; + tensor filled_window_11 = add(x = var_579, y = var_580)[name = tensor("filled_window_11")]; + tensor var_582 = greater_equal(x = filled_11, y = var_17)[name = tensor("op_582")]; + tensor var_583_dtype_0 = const()[name = tensor("op_583_dtype_0"), val = tensor("fp32")]; + tensor var_584 = const()[name = tensor("op_584"), val = tensor([1, 1, 1])]; + tensor var_583 = cast(dtype = var_583_dtype_0, x = var_582)[name = tensor("cast_180")]; + tensor use_roll_11 = reshape(shape = var_584, x = var_583)[name = tensor("use_roll_11")]; + tensor var_586 = sub(x = var_18, y = use_roll_11)[name = tensor("op_586")]; + tensor var_587 = mul(x = filled_window_11, y = var_586)[name = tensor("op_587")]; + tensor var_588 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_588")]; + tensor window_15 = add(x = var_587, y = var_588)[name = tensor("window_15")]; + tensor var_590 = const()[name = tensor("op_590"), val = tensor(0x1p+0)]; + tensor var_591 = add(x = filled_11, y = var_590)[name = tensor("op_591")]; + tensor clip_11 = clip(alpha = var_22, beta = var_17, x = var_591)[name = tensor("clip_11")]; + tensor var_594_begin_0 = const()[name = tensor("op_594_begin_0"), val = tensor([0, 1, 0])]; + tensor var_594_end_0 = const()[name = tensor("op_594_end_0"), val = tensor([1, 2, 256])]; + tensor var_594_end_mask_0 = const()[name = tensor("op_594_end_mask_0"), val = tensor([true, false, true])]; + tensor var_594 = slice_by_index(begin = var_594_begin_0, end = var_594_end_0, end_mask = var_594_end_mask_0, x = x_9)[name = tensor("op_594")]; + tensor var_597_begin_0 = const()[name = tensor("op_597_begin_0"), val = tensor([0, 1, 0])]; + tensor var_597_end_0 = const()[name = tensor("op_597_end_0"), val = tensor([1, 16, 256])]; + tensor var_597_end_mask_0 = const()[name = tensor("op_597_end_mask_0"), val = tensor([true, true, true])]; + tensor var_597 = slice_by_index(begin = var_597_begin_0, end = var_597_end_0, end_mask = var_597_end_mask_0, x = window_15)[name = tensor("op_597")]; tensor rolled_13_interleave_0 = const()[name = tensor("rolled_13_interleave_0"), val = tensor(false)]; - tensor rolled_13 = concat(axis = var_34, interleave = rolled_13_interleave_0, values = (var_866, var_863))[name = tensor("rolled_13")]; + tensor rolled_13 = concat(axis = var_34, interleave = rolled_13_interleave_0, values = (var_597, var_594))[name = tensor("rolled_13")]; tensor clip_12 = clip(alpha = var_22, beta = var_21, x = clip_11)[name = tensor("clip_12")]; - tensor var_871 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_871")]; - tensor var_872_dtype_0 = const()[name = tensor("op_872_dtype_0"), val = tensor("fp32")]; - tensor var_873 = const()[name = tensor("op_873"), val = tensor([1, 16, 1])]; - tensor var_872 = cast(dtype = var_872_dtype_0, x = var_871)[name = tensor("cast_179")]; - tensor fill_mask_13 = reshape(shape = var_873, x = var_872)[name = tensor("fill_mask_13")]; + tensor var_602 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_602")]; + tensor var_603_dtype_0 = const()[name = tensor("op_603_dtype_0"), val = tensor("fp32")]; + tensor var_604 = const()[name = tensor("op_604"), val = tensor([1, 16, 1])]; + tensor var_603 = cast(dtype = var_603_dtype_0, x = var_602)[name = tensor("cast_179")]; + tensor fill_mask_13 = reshape(shape = var_604, x = var_603)[name = tensor("fill_mask_13")]; tensor x_expanded_13_reps_0 = const()[name = tensor("x_expanded_13_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_863)[name = tensor("x_expanded_13")]; - tensor var_877 = sub(x = var_18, y = fill_mask_13)[name = tensor("op_877")]; - tensor var_878 = mul(x = input_101, y = var_877)[name = tensor("op_878")]; - tensor var_879 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_879")]; - tensor filled_window_13 = add(x = var_878, y = var_879)[name = tensor("filled_window_13")]; - tensor var_881 = greater_equal(x = clip_11, y = var_17)[name = tensor("op_881")]; - tensor var_882_dtype_0 = const()[name = tensor("op_882_dtype_0"), val = tensor("fp32")]; - tensor var_883 = const()[name = tensor("op_883"), val = tensor([1, 1, 1])]; - tensor var_882 = cast(dtype = var_882_dtype_0, x = var_881)[name = tensor("cast_178")]; - tensor use_roll_13 = reshape(shape = var_883, x = var_882)[name = tensor("use_roll_13")]; - tensor var_885 = sub(x = var_18, y = use_roll_13)[name = tensor("op_885")]; - tensor var_886 = mul(x = filled_window_13, y = var_885)[name = tensor("op_886")]; - tensor var_887 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_887")]; - tensor input_111 = add(x = var_886, y = var_887)[name = tensor("input_111")]; - tensor var_889 = const()[name = tensor("op_889"), val = tensor(0x1p+0)]; - tensor var_890 = add(x = clip_11, y = var_889)[name = tensor("op_890")]; - tensor clip_13 = clip(alpha = var_22, beta = var_17, x = var_890)[name = tensor("clip_13")]; - tensor x_21_axes_0 = const()[name = tensor("x_21_axes_0"), val = tensor([-1])]; - tensor x_21 = layer_norm(axes = x_21_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_111)[name = tensor("x_21")]; - tensor input_113_perm_0 = const()[name = tensor("input_113_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_43_pad_type_0 = const()[name = tensor("inputs_43_pad_type_0"), val = tensor("valid")]; - tensor inputs_43_strides_0 = const()[name = tensor("inputs_43_strides_0"), val = tensor([1])]; - tensor inputs_43_pad_0 = const()[name = tensor("inputs_43_pad_0"), val = tensor([0, 0])]; - tensor inputs_43_dilations_0 = const()[name = tensor("inputs_43_dilations_0"), val = tensor([1])]; - tensor inputs_43_groups_0 = const()[name = tensor("inputs_43_groups_0"), val = tensor(1)]; - tensor input_113 = transpose(perm = input_113_perm_0, x = x_21)[name = tensor("transpose_66")]; - tensor inputs_43 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_43_dilations_0, groups = inputs_43_groups_0, pad = inputs_43_pad_0, pad_type = inputs_43_pad_type_0, strides = inputs_43_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_113)[name = tensor("inputs_43")]; - tensor var_911_split_sizes_0 = const()[name = tensor("op_911_split_sizes_0"), val = tensor([256, 256])]; - tensor var_911_axis_0 = const()[name = tensor("op_911_axis_0"), val = tensor(1)]; - tensor var_911_0, tensor var_911_1 = split(axis = var_911_axis_0, split_sizes = var_911_split_sizes_0, x = inputs_43)[name = tensor("op_911")]; - tensor var_913 = sigmoid(x = var_911_1)[name = tensor("op_913")]; - tensor inputs_45 = mul(x = var_911_0, y = var_913)[name = tensor("inputs_45")]; - tensor outputs_aug_13_pad_type_0 = const()[name = tensor("outputs_aug_13_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_13_pad_0 = const()[name = tensor("outputs_aug_13_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_13_groups_0 = const()[name = tensor("outputs_aug_13_groups_0"), val = tensor(256)]; - tensor outputs_aug_13_strides_0 = const()[name = tensor("outputs_aug_13_strides_0"), val = tensor([1])]; - tensor outputs_aug_13_dilations_0 = const()[name = tensor("outputs_aug_13_dilations_0"), val = tensor([1])]; - tensor outputs_aug_13 = conv(dilations = outputs_aug_13_dilations_0, groups = outputs_aug_13_groups_0, pad = outputs_aug_13_pad_0, pad_type = outputs_aug_13_pad_type_0, strides = outputs_aug_13_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_45)[name = tensor("outputs_aug_13")]; - tensor input_115_begin_0 = const()[name = tensor("input_115_begin_0"), val = tensor([0, 0, 0])]; - tensor input_115_end_0 = const()[name = tensor("input_115_end_0"), val = tensor([1, 256, 16])]; - tensor input_115_end_mask_0 = const()[name = tensor("input_115_end_mask_0"), val = tensor([true, true, false])]; - tensor input_115 = slice_by_index(begin = input_115_begin_0, end = input_115_end_0, end_mask = input_115_end_mask_0, x = outputs_aug_13)[name = tensor("input_115")]; - tensor inputs_47 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_115)[name = tensor("inputs_47")]; - tensor input_117 = silu(x = inputs_47)[name = tensor("input_117")]; - tensor input_119_pad_type_0 = const()[name = tensor("input_119_pad_type_0"), val = tensor("valid")]; - tensor input_119_strides_0 = const()[name = tensor("input_119_strides_0"), val = tensor([1])]; - tensor input_119_pad_0 = const()[name = tensor("input_119_pad_0"), val = tensor([0, 0])]; - tensor input_119_dilations_0 = const()[name = tensor("input_119_dilations_0"), val = tensor([1])]; - tensor input_119_groups_0 = const()[name = tensor("input_119_groups_0"), val = tensor(1)]; - tensor input_119 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_119_dilations_0, groups = input_119_groups_0, pad = input_119_pad_0, pad_type = input_119_pad_type_0, strides = input_119_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_117)[name = tensor("input_119")]; - tensor conv_out_13_perm_0 = const()[name = tensor("conv_out_13_perm_0"), val = tensor([0, 2, 1])]; - tensor var_944_begin_0 = const()[name = tensor("op_944_begin_0"), val = tensor([0, -1, 0])]; - tensor var_944_end_0 = const()[name = tensor("op_944_end_0"), val = tensor([1, 16, 256])]; - tensor var_944_end_mask_0 = const()[name = tensor("op_944_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_13 = transpose(perm = conv_out_13_perm_0, x = input_119)[name = tensor("transpose_65")]; - tensor var_944 = slice_by_index(begin = var_944_begin_0, end = var_944_end_0, end_mask = var_944_end_mask_0, x = conv_out_13)[name = tensor("op_944")]; - tensor var_947_begin_0 = const()[name = tensor("op_947_begin_0"), val = tensor([0, 2, 0])]; - tensor var_947_end_0 = const()[name = tensor("op_947_end_0"), val = tensor([1, 3, 256])]; - tensor var_947_end_mask_0 = const()[name = tensor("op_947_end_mask_0"), val = tensor([true, false, true])]; - tensor var_947 = slice_by_index(begin = var_947_begin_0, end = var_947_end_0, end_mask = var_947_end_mask_0, x = x_17)[name = tensor("op_947")]; - tensor var_950_begin_0 = const()[name = tensor("op_950_begin_0"), val = tensor([0, 1, 0])]; - tensor var_950_end_0 = const()[name = tensor("op_950_end_0"), val = tensor([1, 16, 256])]; - tensor var_950_end_mask_0 = const()[name = tensor("op_950_end_mask_0"), val = tensor([true, true, true])]; - tensor var_950 = slice_by_index(begin = var_950_begin_0, end = var_950_end_0, end_mask = var_950_end_mask_0, x = input_111)[name = tensor("op_950")]; + tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_594)[name = tensor("x_expanded_13")]; + tensor var_608 = sub(x = var_18, y = fill_mask_13)[name = tensor("op_608")]; + tensor var_609 = mul(x = window_15, y = var_608)[name = tensor("op_609")]; + tensor var_610 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_610")]; + tensor filled_window_13 = add(x = var_609, y = var_610)[name = tensor("filled_window_13")]; + tensor var_612 = greater_equal(x = clip_11, y = var_17)[name = tensor("op_612")]; + tensor var_613_dtype_0 = const()[name = tensor("op_613_dtype_0"), val = tensor("fp32")]; + tensor var_614 = const()[name = tensor("op_614"), val = tensor([1, 1, 1])]; + tensor var_613 = cast(dtype = var_613_dtype_0, x = var_612)[name = tensor("cast_178")]; + tensor use_roll_13 = reshape(shape = var_614, x = var_613)[name = tensor("use_roll_13")]; + tensor var_616 = sub(x = var_18, y = use_roll_13)[name = tensor("op_616")]; + tensor var_617 = mul(x = filled_window_13, y = var_616)[name = tensor("op_617")]; + tensor var_618 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_618")]; + tensor window_17 = add(x = var_617, y = var_618)[name = tensor("window_17")]; + tensor var_620 = const()[name = tensor("op_620"), val = tensor(0x1p+0)]; + tensor var_621 = add(x = clip_11, y = var_620)[name = tensor("op_621")]; + tensor clip_13 = clip(alpha = var_22, beta = var_17, x = var_621)[name = tensor("clip_13")]; + tensor var_624_begin_0 = const()[name = tensor("op_624_begin_0"), val = tensor([0, 2, 0])]; + tensor var_624_end_0 = const()[name = tensor("op_624_end_0"), val = tensor([1, 3, 256])]; + tensor var_624_end_mask_0 = const()[name = tensor("op_624_end_mask_0"), val = tensor([true, false, true])]; + tensor var_624 = slice_by_index(begin = var_624_begin_0, end = var_624_end_0, end_mask = var_624_end_mask_0, x = x_9)[name = tensor("op_624")]; + tensor var_627_begin_0 = const()[name = tensor("op_627_begin_0"), val = tensor([0, 1, 0])]; + tensor var_627_end_0 = const()[name = tensor("op_627_end_0"), val = tensor([1, 16, 256])]; + tensor var_627_end_mask_0 = const()[name = tensor("op_627_end_mask_0"), val = tensor([true, true, true])]; + tensor var_627 = slice_by_index(begin = var_627_begin_0, end = var_627_end_0, end_mask = var_627_end_mask_0, x = window_17)[name = tensor("op_627")]; tensor rolled_15_interleave_0 = const()[name = tensor("rolled_15_interleave_0"), val = tensor(false)]; - tensor rolled_15 = concat(axis = var_34, interleave = rolled_15_interleave_0, values = (var_950, var_947))[name = tensor("rolled_15")]; + tensor rolled_15 = concat(axis = var_34, interleave = rolled_15_interleave_0, values = (var_627, var_624))[name = tensor("rolled_15")]; tensor clip_14 = clip(alpha = var_22, beta = var_21, x = clip_13)[name = tensor("clip_14")]; - tensor var_955 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_955")]; - tensor var_956_dtype_0 = const()[name = tensor("op_956_dtype_0"), val = tensor("fp32")]; - tensor var_957 = const()[name = tensor("op_957"), val = tensor([1, 16, 1])]; - tensor var_956 = cast(dtype = var_956_dtype_0, x = var_955)[name = tensor("cast_177")]; - tensor fill_mask_15 = reshape(shape = var_957, x = var_956)[name = tensor("fill_mask_15")]; + tensor var_632 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_632")]; + tensor var_633_dtype_0 = const()[name = tensor("op_633_dtype_0"), val = tensor("fp32")]; + tensor var_634 = const()[name = tensor("op_634"), val = tensor([1, 16, 1])]; + tensor var_633 = cast(dtype = var_633_dtype_0, x = var_632)[name = tensor("cast_177")]; + tensor fill_mask_15 = reshape(shape = var_634, x = var_633)[name = tensor("fill_mask_15")]; tensor x_expanded_15_reps_0 = const()[name = tensor("x_expanded_15_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_947)[name = tensor("x_expanded_15")]; - tensor var_961 = sub(x = var_18, y = fill_mask_15)[name = tensor("op_961")]; - tensor var_962 = mul(x = input_111, y = var_961)[name = tensor("op_962")]; - tensor var_963 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_963")]; - tensor filled_window_15 = add(x = var_962, y = var_963)[name = tensor("filled_window_15")]; - tensor var_965 = greater_equal(x = clip_13, y = var_17)[name = tensor("op_965")]; - tensor var_966_dtype_0 = const()[name = tensor("op_966_dtype_0"), val = tensor("fp32")]; - tensor var_967 = const()[name = tensor("op_967"), val = tensor([1, 1, 1])]; - tensor var_966 = cast(dtype = var_966_dtype_0, x = var_965)[name = tensor("cast_176")]; - tensor use_roll_15 = reshape(shape = var_967, x = var_966)[name = tensor("use_roll_15")]; - tensor var_969 = sub(x = var_18, y = use_roll_15)[name = tensor("op_969")]; - tensor var_970 = mul(x = filled_window_15, y = var_969)[name = tensor("op_970")]; - tensor var_971 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_971")]; - tensor input_121 = add(x = var_970, y = var_971)[name = tensor("input_121")]; - tensor var_973 = const()[name = tensor("op_973"), val = tensor(0x1p+0)]; - tensor var_974 = add(x = clip_13, y = var_973)[name = tensor("op_974")]; - tensor clip_15 = clip(alpha = var_22, beta = var_17, x = var_974)[name = tensor("clip_15")]; - tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; - tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_121)[name = tensor("x_23")]; - tensor input_123_perm_0 = const()[name = tensor("input_123_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_49_pad_type_0 = const()[name = tensor("inputs_49_pad_type_0"), val = tensor("valid")]; - tensor inputs_49_strides_0 = const()[name = tensor("inputs_49_strides_0"), val = tensor([1])]; - tensor inputs_49_pad_0 = const()[name = tensor("inputs_49_pad_0"), val = tensor([0, 0])]; - tensor inputs_49_dilations_0 = const()[name = tensor("inputs_49_dilations_0"), val = tensor([1])]; - tensor inputs_49_groups_0 = const()[name = tensor("inputs_49_groups_0"), val = tensor(1)]; - tensor input_123 = transpose(perm = input_123_perm_0, x = x_23)[name = tensor("transpose_64")]; - tensor inputs_49 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_49_dilations_0, groups = inputs_49_groups_0, pad = inputs_49_pad_0, pad_type = inputs_49_pad_type_0, strides = inputs_49_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_123)[name = tensor("inputs_49")]; - tensor var_995_split_sizes_0 = const()[name = tensor("op_995_split_sizes_0"), val = tensor([256, 256])]; - tensor var_995_axis_0 = const()[name = tensor("op_995_axis_0"), val = tensor(1)]; - tensor var_995_0, tensor var_995_1 = split(axis = var_995_axis_0, split_sizes = var_995_split_sizes_0, x = inputs_49)[name = tensor("op_995")]; - tensor var_997 = sigmoid(x = var_995_1)[name = tensor("op_997")]; - tensor inputs_51 = mul(x = var_995_0, y = var_997)[name = tensor("inputs_51")]; - tensor outputs_aug_15_pad_type_0 = const()[name = tensor("outputs_aug_15_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_15_pad_0 = const()[name = tensor("outputs_aug_15_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_15_groups_0 = const()[name = tensor("outputs_aug_15_groups_0"), val = tensor(256)]; - tensor outputs_aug_15_strides_0 = const()[name = tensor("outputs_aug_15_strides_0"), val = tensor([1])]; - tensor outputs_aug_15_dilations_0 = const()[name = tensor("outputs_aug_15_dilations_0"), val = tensor([1])]; - tensor outputs_aug_15 = conv(dilations = outputs_aug_15_dilations_0, groups = outputs_aug_15_groups_0, pad = outputs_aug_15_pad_0, pad_type = outputs_aug_15_pad_type_0, strides = outputs_aug_15_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_51)[name = tensor("outputs_aug_15")]; - tensor input_125_begin_0 = const()[name = tensor("input_125_begin_0"), val = tensor([0, 0, 0])]; - tensor input_125_end_0 = const()[name = tensor("input_125_end_0"), val = tensor([1, 256, 16])]; - tensor input_125_end_mask_0 = const()[name = tensor("input_125_end_mask_0"), val = tensor([true, true, false])]; - tensor input_125 = slice_by_index(begin = input_125_begin_0, end = input_125_end_0, end_mask = input_125_end_mask_0, x = outputs_aug_15)[name = tensor("input_125")]; - tensor inputs_53 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_125)[name = tensor("inputs_53")]; - tensor input_127 = silu(x = inputs_53)[name = tensor("input_127")]; - tensor input_129_pad_type_0 = const()[name = tensor("input_129_pad_type_0"), val = tensor("valid")]; - tensor input_129_strides_0 = const()[name = tensor("input_129_strides_0"), val = tensor([1])]; - tensor input_129_pad_0 = const()[name = tensor("input_129_pad_0"), val = tensor([0, 0])]; - tensor input_129_dilations_0 = const()[name = tensor("input_129_dilations_0"), val = tensor([1])]; - tensor input_129_groups_0 = const()[name = tensor("input_129_groups_0"), val = tensor(1)]; - tensor input_129 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_129_dilations_0, groups = input_129_groups_0, pad = input_129_pad_0, pad_type = input_129_pad_type_0, strides = input_129_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_127)[name = tensor("input_129")]; - tensor conv_out_15_perm_0 = const()[name = tensor("conv_out_15_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1028_begin_0 = const()[name = tensor("op_1028_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1028_end_0 = const()[name = tensor("op_1028_end_0"), val = tensor([1, 16, 256])]; - tensor var_1028_end_mask_0 = const()[name = tensor("op_1028_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_15 = transpose(perm = conv_out_15_perm_0, x = input_129)[name = tensor("transpose_63")]; - tensor var_1028 = slice_by_index(begin = var_1028_begin_0, end = var_1028_end_0, end_mask = var_1028_end_mask_0, x = conv_out_15)[name = tensor("op_1028")]; - tensor var_1031_begin_0 = const()[name = tensor("op_1031_begin_0"), val = tensor([0, 3, 0])]; - tensor var_1031_end_0 = const()[name = tensor("op_1031_end_0"), val = tensor([1, 4, 256])]; - tensor var_1031_end_mask_0 = const()[name = tensor("op_1031_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1031 = slice_by_index(begin = var_1031_begin_0, end = var_1031_end_0, end_mask = var_1031_end_mask_0, x = x_17)[name = tensor("op_1031")]; - tensor var_1034_begin_0 = const()[name = tensor("op_1034_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1034_end_0 = const()[name = tensor("op_1034_end_0"), val = tensor([1, 16, 256])]; - tensor var_1034_end_mask_0 = const()[name = tensor("op_1034_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1034 = slice_by_index(begin = var_1034_begin_0, end = var_1034_end_0, end_mask = var_1034_end_mask_0, x = input_121)[name = tensor("op_1034")]; + tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_624)[name = tensor("x_expanded_15")]; + tensor var_638 = sub(x = var_18, y = fill_mask_15)[name = tensor("op_638")]; + tensor var_639 = mul(x = window_17, y = var_638)[name = tensor("op_639")]; + tensor var_640 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_640")]; + tensor filled_window_15 = add(x = var_639, y = var_640)[name = tensor("filled_window_15")]; + tensor var_642 = greater_equal(x = clip_13, y = var_17)[name = tensor("op_642")]; + tensor var_643_dtype_0 = const()[name = tensor("op_643_dtype_0"), val = tensor("fp32")]; + tensor var_644 = const()[name = tensor("op_644"), val = tensor([1, 1, 1])]; + tensor var_643 = cast(dtype = var_643_dtype_0, x = var_642)[name = tensor("cast_176")]; + tensor use_roll_15 = reshape(shape = var_644, x = var_643)[name = tensor("use_roll_15")]; + tensor var_646 = sub(x = var_18, y = use_roll_15)[name = tensor("op_646")]; + tensor var_647 = mul(x = filled_window_15, y = var_646)[name = tensor("op_647")]; + tensor var_648 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_648")]; + tensor window_19 = add(x = var_647, y = var_648)[name = tensor("window_19")]; + tensor var_650 = const()[name = tensor("op_650"), val = tensor(0x1p+0)]; + tensor var_651 = add(x = clip_13, y = var_650)[name = tensor("op_651")]; + tensor clip_15 = clip(alpha = var_22, beta = var_17, x = var_651)[name = tensor("clip_15")]; + tensor var_654_begin_0 = const()[name = tensor("op_654_begin_0"), val = tensor([0, 3, 0])]; + tensor var_654_end_0 = const()[name = tensor("op_654_end_0"), val = tensor([1, 4, 256])]; + tensor var_654_end_mask_0 = const()[name = tensor("op_654_end_mask_0"), val = tensor([true, false, true])]; + tensor var_654 = slice_by_index(begin = var_654_begin_0, end = var_654_end_0, end_mask = var_654_end_mask_0, x = x_9)[name = tensor("op_654")]; + tensor var_657_begin_0 = const()[name = tensor("op_657_begin_0"), val = tensor([0, 1, 0])]; + tensor var_657_end_0 = const()[name = tensor("op_657_end_0"), val = tensor([1, 16, 256])]; + tensor var_657_end_mask_0 = const()[name = tensor("op_657_end_mask_0"), val = tensor([true, true, true])]; + tensor var_657 = slice_by_index(begin = var_657_begin_0, end = var_657_end_0, end_mask = var_657_end_mask_0, x = window_19)[name = tensor("op_657")]; tensor rolled_17_interleave_0 = const()[name = tensor("rolled_17_interleave_0"), val = tensor(false)]; - tensor rolled_17 = concat(axis = var_34, interleave = rolled_17_interleave_0, values = (var_1034, var_1031))[name = tensor("rolled_17")]; + tensor rolled_17 = concat(axis = var_34, interleave = rolled_17_interleave_0, values = (var_657, var_654))[name = tensor("rolled_17")]; tensor clip_16 = clip(alpha = var_22, beta = var_21, x = clip_15)[name = tensor("clip_16")]; - tensor var_1039 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_1039")]; - tensor var_1040_dtype_0 = const()[name = tensor("op_1040_dtype_0"), val = tensor("fp32")]; - tensor var_1041 = const()[name = tensor("op_1041"), val = tensor([1, 16, 1])]; - tensor var_1040 = cast(dtype = var_1040_dtype_0, x = var_1039)[name = tensor("cast_175")]; - tensor fill_mask_17 = reshape(shape = var_1041, x = var_1040)[name = tensor("fill_mask_17")]; + tensor var_662 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_662")]; + tensor var_663_dtype_0 = const()[name = tensor("op_663_dtype_0"), val = tensor("fp32")]; + tensor var_664 = const()[name = tensor("op_664"), val = tensor([1, 16, 1])]; + tensor var_663 = cast(dtype = var_663_dtype_0, x = var_662)[name = tensor("cast_175")]; + tensor fill_mask_17 = reshape(shape = var_664, x = var_663)[name = tensor("fill_mask_17")]; tensor x_expanded_17_reps_0 = const()[name = tensor("x_expanded_17_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_1031)[name = tensor("x_expanded_17")]; - tensor var_1045 = sub(x = var_18, y = fill_mask_17)[name = tensor("op_1045")]; - tensor var_1046 = mul(x = input_121, y = var_1045)[name = tensor("op_1046")]; - tensor var_1047 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_1047")]; - tensor filled_window_17 = add(x = var_1046, y = var_1047)[name = tensor("filled_window_17")]; - tensor var_1049 = greater_equal(x = clip_15, y = var_17)[name = tensor("op_1049")]; - tensor var_1050_dtype_0 = const()[name = tensor("op_1050_dtype_0"), val = tensor("fp32")]; - tensor var_1051 = const()[name = tensor("op_1051"), val = tensor([1, 1, 1])]; - tensor var_1050 = cast(dtype = var_1050_dtype_0, x = var_1049)[name = tensor("cast_174")]; - tensor use_roll_17 = reshape(shape = var_1051, x = var_1050)[name = tensor("use_roll_17")]; - tensor var_1053 = sub(x = var_18, y = use_roll_17)[name = tensor("op_1053")]; - tensor var_1054 = mul(x = filled_window_17, y = var_1053)[name = tensor("op_1054")]; - tensor var_1055 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_1055")]; - tensor input_131 = add(x = var_1054, y = var_1055)[name = tensor("input_131")]; - tensor var_1057 = const()[name = tensor("op_1057"), val = tensor(0x1p+0)]; - tensor var_1058 = add(x = clip_15, y = var_1057)[name = tensor("op_1058")]; - tensor clip_17 = clip(alpha = var_22, beta = var_17, x = var_1058)[name = tensor("clip_17")]; - tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; - tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_131)[name = tensor("x_25")]; - tensor input_133_perm_0 = const()[name = tensor("input_133_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_55_pad_type_0 = const()[name = tensor("inputs_55_pad_type_0"), val = tensor("valid")]; - tensor inputs_55_strides_0 = const()[name = tensor("inputs_55_strides_0"), val = tensor([1])]; - tensor inputs_55_pad_0 = const()[name = tensor("inputs_55_pad_0"), val = tensor([0, 0])]; - tensor inputs_55_dilations_0 = const()[name = tensor("inputs_55_dilations_0"), val = tensor([1])]; - tensor inputs_55_groups_0 = const()[name = tensor("inputs_55_groups_0"), val = tensor(1)]; - tensor input_133 = transpose(perm = input_133_perm_0, x = x_25)[name = tensor("transpose_62")]; - tensor inputs_55 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_55_dilations_0, groups = inputs_55_groups_0, pad = inputs_55_pad_0, pad_type = inputs_55_pad_type_0, strides = inputs_55_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_133)[name = tensor("inputs_55")]; - tensor var_1079_split_sizes_0 = const()[name = tensor("op_1079_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1079_axis_0 = const()[name = tensor("op_1079_axis_0"), val = tensor(1)]; - tensor var_1079_0, tensor var_1079_1 = split(axis = var_1079_axis_0, split_sizes = var_1079_split_sizes_0, x = inputs_55)[name = tensor("op_1079")]; - tensor var_1081 = sigmoid(x = var_1079_1)[name = tensor("op_1081")]; - tensor inputs_57 = mul(x = var_1079_0, y = var_1081)[name = tensor("inputs_57")]; - tensor outputs_aug_17_pad_type_0 = const()[name = tensor("outputs_aug_17_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_17_pad_0 = const()[name = tensor("outputs_aug_17_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_17_groups_0 = const()[name = tensor("outputs_aug_17_groups_0"), val = tensor(256)]; - tensor outputs_aug_17_strides_0 = const()[name = tensor("outputs_aug_17_strides_0"), val = tensor([1])]; - tensor outputs_aug_17_dilations_0 = const()[name = tensor("outputs_aug_17_dilations_0"), val = tensor([1])]; - tensor outputs_aug_17 = conv(dilations = outputs_aug_17_dilations_0, groups = outputs_aug_17_groups_0, pad = outputs_aug_17_pad_0, pad_type = outputs_aug_17_pad_type_0, strides = outputs_aug_17_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_57)[name = tensor("outputs_aug_17")]; - tensor input_135_begin_0 = const()[name = tensor("input_135_begin_0"), val = tensor([0, 0, 0])]; - tensor input_135_end_0 = const()[name = tensor("input_135_end_0"), val = tensor([1, 256, 16])]; - tensor input_135_end_mask_0 = const()[name = tensor("input_135_end_mask_0"), val = tensor([true, true, false])]; - tensor input_135 = slice_by_index(begin = input_135_begin_0, end = input_135_end_0, end_mask = input_135_end_mask_0, x = outputs_aug_17)[name = tensor("input_135")]; - tensor inputs_59 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_135)[name = tensor("inputs_59")]; - tensor input_137 = silu(x = inputs_59)[name = tensor("input_137")]; - tensor input_139_pad_type_0 = const()[name = tensor("input_139_pad_type_0"), val = tensor("valid")]; - tensor input_139_strides_0 = const()[name = tensor("input_139_strides_0"), val = tensor([1])]; - tensor input_139_pad_0 = const()[name = tensor("input_139_pad_0"), val = tensor([0, 0])]; - tensor input_139_dilations_0 = const()[name = tensor("input_139_dilations_0"), val = tensor([1])]; - tensor input_139_groups_0 = const()[name = tensor("input_139_groups_0"), val = tensor(1)]; - tensor input_139 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_139_dilations_0, groups = input_139_groups_0, pad = input_139_pad_0, pad_type = input_139_pad_type_0, strides = input_139_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_137)[name = tensor("input_139")]; - tensor conv_out_17_perm_0 = const()[name = tensor("conv_out_17_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1112_begin_0 = const()[name = tensor("op_1112_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1112_end_0 = const()[name = tensor("op_1112_end_0"), val = tensor([1, 16, 256])]; - tensor var_1112_end_mask_0 = const()[name = tensor("op_1112_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_17 = transpose(perm = conv_out_17_perm_0, x = input_139)[name = tensor("transpose_61")]; - tensor var_1112 = slice_by_index(begin = var_1112_begin_0, end = var_1112_end_0, end_mask = var_1112_end_mask_0, x = conv_out_17)[name = tensor("op_1112")]; - tensor var_1115_begin_0 = const()[name = tensor("op_1115_begin_0"), val = tensor([0, 4, 0])]; - tensor var_1115_end_0 = const()[name = tensor("op_1115_end_0"), val = tensor([1, 1, 256])]; - tensor var_1115_end_mask_0 = const()[name = tensor("op_1115_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1115 = slice_by_index(begin = var_1115_begin_0, end = var_1115_end_0, end_mask = var_1115_end_mask_0, x = x_17)[name = tensor("op_1115")]; - tensor var_1118_begin_0 = const()[name = tensor("op_1118_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1118_end_0 = const()[name = tensor("op_1118_end_0"), val = tensor([1, 16, 256])]; - tensor var_1118_end_mask_0 = const()[name = tensor("op_1118_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1118 = slice_by_index(begin = var_1118_begin_0, end = var_1118_end_0, end_mask = var_1118_end_mask_0, x = input_131)[name = tensor("op_1118")]; + tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_654)[name = tensor("x_expanded_17")]; + tensor var_668 = sub(x = var_18, y = fill_mask_17)[name = tensor("op_668")]; + tensor var_669 = mul(x = window_19, y = var_668)[name = tensor("op_669")]; + tensor var_670 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_670")]; + tensor filled_window_17 = add(x = var_669, y = var_670)[name = tensor("filled_window_17")]; + tensor var_672 = greater_equal(x = clip_15, y = var_17)[name = tensor("op_672")]; + tensor var_673_dtype_0 = const()[name = tensor("op_673_dtype_0"), val = tensor("fp32")]; + tensor var_674 = const()[name = tensor("op_674"), val = tensor([1, 1, 1])]; + tensor var_673 = cast(dtype = var_673_dtype_0, x = var_672)[name = tensor("cast_174")]; + tensor use_roll_17 = reshape(shape = var_674, x = var_673)[name = tensor("use_roll_17")]; + tensor var_676 = sub(x = var_18, y = use_roll_17)[name = tensor("op_676")]; + tensor var_677 = mul(x = filled_window_17, y = var_676)[name = tensor("op_677")]; + tensor var_678 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_678")]; + tensor window_21 = add(x = var_677, y = var_678)[name = tensor("window_21")]; + tensor var_680 = const()[name = tensor("op_680"), val = tensor(0x1p+0)]; + tensor var_681 = add(x = clip_15, y = var_680)[name = tensor("op_681")]; + tensor clip_17 = clip(alpha = var_22, beta = var_17, x = var_681)[name = tensor("clip_17")]; + tensor var_684_begin_0 = const()[name = tensor("op_684_begin_0"), val = tensor([0, 4, 0])]; + tensor var_684_end_0 = const()[name = tensor("op_684_end_0"), val = tensor([1, 1, 256])]; + tensor var_684_end_mask_0 = const()[name = tensor("op_684_end_mask_0"), val = tensor([true, true, true])]; + tensor var_684 = slice_by_index(begin = var_684_begin_0, end = var_684_end_0, end_mask = var_684_end_mask_0, x = x_9)[name = tensor("op_684")]; + tensor var_687_begin_0 = const()[name = tensor("op_687_begin_0"), val = tensor([0, 1, 0])]; + tensor var_687_end_0 = const()[name = tensor("op_687_end_0"), val = tensor([1, 16, 256])]; + tensor var_687_end_mask_0 = const()[name = tensor("op_687_end_mask_0"), val = tensor([true, true, true])]; + tensor var_687 = slice_by_index(begin = var_687_begin_0, end = var_687_end_0, end_mask = var_687_end_mask_0, x = window_21)[name = tensor("op_687")]; tensor rolled_19_interleave_0 = const()[name = tensor("rolled_19_interleave_0"), val = tensor(false)]; - tensor rolled_19 = concat(axis = var_34, interleave = rolled_19_interleave_0, values = (var_1118, var_1115))[name = tensor("rolled_19")]; + tensor rolled_19 = concat(axis = var_34, interleave = rolled_19_interleave_0, values = (var_687, var_684))[name = tensor("rolled_19")]; tensor clip_18 = clip(alpha = var_22, beta = var_21, x = clip_17)[name = tensor("clip_18")]; - tensor var_1123 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_1123")]; - tensor var_1124_dtype_0 = const()[name = tensor("op_1124_dtype_0"), val = tensor("fp32")]; - tensor var_1125 = const()[name = tensor("op_1125"), val = tensor([1, 16, 1])]; - tensor var_1124 = cast(dtype = var_1124_dtype_0, x = var_1123)[name = tensor("cast_173")]; - tensor fill_mask_19 = reshape(shape = var_1125, x = var_1124)[name = tensor("fill_mask_19")]; + tensor var_692 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_692")]; + tensor var_693_dtype_0 = const()[name = tensor("op_693_dtype_0"), val = tensor("fp32")]; + tensor var_694 = const()[name = tensor("op_694"), val = tensor([1, 16, 1])]; + tensor var_693 = cast(dtype = var_693_dtype_0, x = var_692)[name = tensor("cast_173")]; + tensor fill_mask_19 = reshape(shape = var_694, x = var_693)[name = tensor("fill_mask_19")]; tensor x_expanded_19_reps_0 = const()[name = tensor("x_expanded_19_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_1115)[name = tensor("x_expanded_19")]; - tensor var_1129 = sub(x = var_18, y = fill_mask_19)[name = tensor("op_1129")]; - tensor var_1130 = mul(x = input_131, y = var_1129)[name = tensor("op_1130")]; - tensor var_1131 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_1131")]; - tensor filled_window_19 = add(x = var_1130, y = var_1131)[name = tensor("filled_window_19")]; - tensor var_1133 = greater_equal(x = clip_17, y = var_17)[name = tensor("op_1133")]; - tensor var_1134_dtype_0 = const()[name = tensor("op_1134_dtype_0"), val = tensor("fp32")]; - tensor var_1135 = const()[name = tensor("op_1135"), val = tensor([1, 1, 1])]; - tensor var_1134 = cast(dtype = var_1134_dtype_0, x = var_1133)[name = tensor("cast_172")]; - tensor use_roll_19 = reshape(shape = var_1135, x = var_1134)[name = tensor("use_roll_19")]; - tensor var_1137 = sub(x = var_18, y = use_roll_19)[name = tensor("op_1137")]; - tensor var_1138 = mul(x = filled_window_19, y = var_1137)[name = tensor("op_1138")]; - tensor var_1139 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_1139")]; - tensor input_141 = add(x = var_1138, y = var_1139)[name = tensor("input_141")]; - tensor var_1141 = const()[name = tensor("op_1141"), val = tensor(0x1p+0)]; - tensor var_1142 = add(x = clip_17, y = var_1141)[name = tensor("op_1142")]; - tensor clip_19 = clip(alpha = var_22, beta = var_17, x = var_1142)[name = tensor("clip_19")]; - tensor x_27_axes_0 = const()[name = tensor("x_27_axes_0"), val = tensor([-1])]; - tensor x_27 = layer_norm(axes = x_27_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_141)[name = tensor("x_27")]; - tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_61_pad_type_0 = const()[name = tensor("inputs_61_pad_type_0"), val = tensor("valid")]; - tensor inputs_61_strides_0 = const()[name = tensor("inputs_61_strides_0"), val = tensor([1])]; - tensor inputs_61_pad_0 = const()[name = tensor("inputs_61_pad_0"), val = tensor([0, 0])]; - tensor inputs_61_dilations_0 = const()[name = tensor("inputs_61_dilations_0"), val = tensor([1])]; - tensor inputs_61_groups_0 = const()[name = tensor("inputs_61_groups_0"), val = tensor(1)]; - tensor input_143 = transpose(perm = input_143_perm_0, x = x_27)[name = tensor("transpose_60")]; - tensor inputs_61 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_61_dilations_0, groups = inputs_61_groups_0, pad = inputs_61_pad_0, pad_type = inputs_61_pad_type_0, strides = inputs_61_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_61")]; - tensor var_1163_split_sizes_0 = const()[name = tensor("op_1163_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1163_axis_0 = const()[name = tensor("op_1163_axis_0"), val = tensor(1)]; - tensor var_1163_0, tensor var_1163_1 = split(axis = var_1163_axis_0, split_sizes = var_1163_split_sizes_0, x = inputs_61)[name = tensor("op_1163")]; - tensor var_1165 = sigmoid(x = var_1163_1)[name = tensor("op_1165")]; - tensor inputs_63 = mul(x = var_1163_0, y = var_1165)[name = tensor("inputs_63")]; - tensor outputs_aug_19_pad_type_0 = const()[name = tensor("outputs_aug_19_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_19_pad_0 = const()[name = tensor("outputs_aug_19_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_19_groups_0 = const()[name = tensor("outputs_aug_19_groups_0"), val = tensor(256)]; - tensor outputs_aug_19_strides_0 = const()[name = tensor("outputs_aug_19_strides_0"), val = tensor([1])]; - tensor outputs_aug_19_dilations_0 = const()[name = tensor("outputs_aug_19_dilations_0"), val = tensor([1])]; - tensor outputs_aug_19 = conv(dilations = outputs_aug_19_dilations_0, groups = outputs_aug_19_groups_0, pad = outputs_aug_19_pad_0, pad_type = outputs_aug_19_pad_type_0, strides = outputs_aug_19_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_63)[name = tensor("outputs_aug_19")]; - tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; - tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([1, 256, 16])]; - tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; - tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug_19)[name = tensor("input_145")]; - tensor inputs_65 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_145)[name = tensor("inputs_65")]; - tensor input_147 = silu(x = inputs_65)[name = tensor("input_147")]; - tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; - tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; - tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; - tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; - tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; - tensor input_149 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; - tensor conv_out_19_perm_0 = const()[name = tensor("conv_out_19_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1196_begin_0 = const()[name = tensor("op_1196_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1196_end_0 = const()[name = tensor("op_1196_end_0"), val = tensor([1, 16, 256])]; - tensor var_1196_end_mask_0 = const()[name = tensor("op_1196_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_19 = transpose(perm = conv_out_19_perm_0, x = input_149)[name = tensor("transpose_59")]; - tensor var_1196 = slice_by_index(begin = var_1196_begin_0, end = var_1196_end_0, end_mask = var_1196_end_mask_0, x = conv_out_19)[name = tensor("op_1196")]; - tensor var_1199_interleave_0 = const()[name = tensor("op_1199_interleave_0"), val = tensor(false)]; - tensor var_1199 = concat(axis = var_34, interleave = var_1199_interleave_0, values = (var_860, var_944, var_1028, var_1112, var_1196))[name = tensor("op_1199")]; - tensor input_151 = add(x = x_17, y = var_1199)[name = tensor("input_151")]; - tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; - tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; - tensor inputs_67 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_17")]; - tensor input_155 = silu(x = inputs_67)[name = tensor("input_155")]; - tensor input_159 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_18")]; - tensor var_1222 = const()[name = tensor("op_1222"), val = tensor(0x1p-1)]; - tensor var_1223 = mul(x = input_159, y = var_1222)[name = tensor("op_1223")]; - tensor input_161 = add(x = var_1223, y = input_151)[name = tensor("input_161")]; - tensor input_163_axes_0 = const()[name = tensor("input_163_axes_0"), val = tensor([-1])]; - tensor input_163 = layer_norm(axes = input_163_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_36, gamma = encoder_layer_norm_1_weight, x = input_161)[name = tensor("input_163")]; - tensor input_165_axes_0 = const()[name = tensor("input_165_axes_0"), val = tensor([-1])]; - tensor input_165 = layer_norm(axes = input_165_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_163)[name = tensor("input_165")]; - tensor inputs_69 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_165)[name = tensor("linear_19")]; - tensor input_167 = silu(x = inputs_69)[name = tensor("input_167")]; - tensor input_171 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_167)[name = tensor("linear_20")]; - tensor var_1252 = const()[name = tensor("op_1252"), val = tensor(0x1p-1)]; - tensor var_1253 = mul(x = input_171, y = var_1252)[name = tensor("op_1253")]; - tensor input_173 = add(x = var_1253, y = input_163)[name = tensor("input_173")]; - tensor x_29_axes_0 = const()[name = tensor("x_29_axes_0"), val = tensor([-1])]; - tensor x_29 = layer_norm(axes = x_29_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_36, gamma = encoder_ret_lns_2_weight, x = input_173)[name = tensor("x_29")]; + tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_684)[name = tensor("x_expanded_19")]; + tensor var_698 = sub(x = var_18, y = fill_mask_19)[name = tensor("op_698")]; + tensor var_699 = mul(x = window_21, y = var_698)[name = tensor("op_699")]; + tensor var_700 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_700")]; + tensor filled_window_19 = add(x = var_699, y = var_700)[name = tensor("filled_window_19")]; + tensor var_702 = greater_equal(x = clip_17, y = var_17)[name = tensor("op_702")]; + tensor var_703_dtype_0 = const()[name = tensor("op_703_dtype_0"), val = tensor("fp32")]; + tensor var_704 = const()[name = tensor("op_704"), val = tensor([1, 1, 1])]; + tensor var_703 = cast(dtype = var_703_dtype_0, x = var_702)[name = tensor("cast_172")]; + tensor use_roll_19 = reshape(shape = var_704, x = var_703)[name = tensor("use_roll_19")]; + tensor var_706 = sub(x = var_18, y = use_roll_19)[name = tensor("op_706")]; + tensor var_707 = mul(x = filled_window_19, y = var_706)[name = tensor("op_707")]; + tensor var_708 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_708")]; + tensor window_23 = add(x = var_707, y = var_708)[name = tensor("window_23")]; + tensor var_710 = const()[name = tensor("op_710"), val = tensor(0x1p+0)]; + tensor var_711 = add(x = clip_17, y = var_710)[name = tensor("op_711")]; + tensor clip_19 = clip(alpha = var_22, beta = var_17, x = var_711)[name = tensor("clip_19")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_31, interleave = input_61_interleave_0, values = (window_15, window_17, window_19, window_21, window_23))[name = tensor("input_61")]; + tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; + tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; + tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_13_pad_type_0 = const()[name = tensor("inputs_13_pad_type_0"), val = tensor("valid")]; + tensor inputs_13_strides_0 = const()[name = tensor("inputs_13_strides_0"), val = tensor([1])]; + tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; + tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; + tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; + tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; + tensor var_734_split_sizes_0 = const()[name = tensor("op_734_split_sizes_0"), val = tensor([256, 256])]; + tensor var_734_axis_0 = const()[name = tensor("op_734_axis_0"), val = tensor(1)]; + tensor var_734_0, tensor var_734_1 = split(axis = var_734_axis_0, split_sizes = var_734_split_sizes_0, x = inputs_13)[name = tensor("op_734")]; + tensor var_736 = sigmoid(x = var_734_1)[name = tensor("op_736")]; + tensor inputs_15 = mul(x = var_734_0, y = var_736)[name = tensor("inputs_15")]; + tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; + tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; + tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; + tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_15)[name = tensor("outputs_aug_3")]; + tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; + tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([5, 256, 16])]; + tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; + tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_3)[name = tensor("input_65")]; + tensor inputs_17 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_65)[name = tensor("inputs_17")]; + tensor input_67 = silu(x = inputs_17)[name = tensor("input_67")]; + tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; + tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; + tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; + tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; + tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; + tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; + tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; + tensor var_767_begin_0 = const()[name = tensor("op_767_begin_0"), val = tensor([0, -1, 0])]; + tensor var_767_end_0 = const()[name = tensor("op_767_end_0"), val = tensor([5, 16, 256])]; + tensor var_767_end_mask_0 = const()[name = tensor("op_767_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_767 = slice_by_index(begin = var_767_begin_0, end = var_767_end_0, end_mask = var_767_end_mask_0, x = conv_out_3)[name = tensor("op_767")]; + tensor var_769_perm_0 = const()[name = tensor("op_769_perm_0"), val = tensor([1, 0, 2])]; + tensor var_769 = transpose(perm = var_769_perm_0, x = var_767)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_769)[name = tensor("input_71")]; + tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; + tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; + tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; + tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; + tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; + tensor var_792 = const()[name = tensor("op_792"), val = tensor(0x1p-1)]; + tensor var_793 = mul(x = input_79, y = var_792)[name = tensor("op_793")]; + tensor input_81 = add(x = var_793, y = input_71)[name = tensor("input_81")]; + tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; + tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_36, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; + tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; + tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; + tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; + tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; + tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; + tensor var_822 = const()[name = tensor("op_822"), val = tensor(0x1p-1)]; + tensor var_823 = mul(x = input_91, y = var_822)[name = tensor("op_823")]; + tensor input_93 = add(x = var_823, y = input_83)[name = tensor("input_93")]; + tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; + tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_36, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; tensor prev_kv_5_end_0 = const()[name = tensor("prev_kv_5_end_0"), val = tensor([3, 1, 4, 64, 64])]; tensor prev_kv_5_end_mask_0 = const()[name = tensor("prev_kv_5_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1175,462 +868,308 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_1267 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_29)[name = tensor("linear_21")]; - tensor var_1268 = const()[name = tensor("op_1268"), val = tensor([1, 5, 4, 64])]; - tensor var_1269 = reshape(shape = var_1268, x = var_1267)[name = tensor("op_1269")]; + tensor var_837 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_838 = const()[name = tensor("op_838"), val = tensor([1, 5, 4, 64])]; + tensor var_839 = reshape(shape = var_838, x = var_837)[name = tensor("op_839")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1273 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_29)[name = tensor("linear_22")]; - tensor var_1274 = const()[name = tensor("op_1274"), val = tensor(0x1p-3)]; - tensor var_1275 = mul(x = var_1273, y = var_1274)[name = tensor("op_1275")]; - tensor var_1276 = const()[name = tensor("op_1276"), val = tensor([1, 5, 4, 64])]; - tensor var_1277 = reshape(shape = var_1276, x = var_1275)[name = tensor("op_1277")]; + tensor var_843 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_844 = const()[name = tensor("op_844"), val = tensor(0x1p-3)]; + tensor var_845 = mul(x = var_843, y = var_844)[name = tensor("op_845")]; + tensor var_846 = const()[name = tensor("op_846"), val = tensor([1, 5, 4, 64])]; + tensor var_847 = reshape(shape = var_846, x = var_845)[name = tensor("op_847")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1281 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_29)[name = tensor("linear_23")]; - tensor var_1282 = const()[name = tensor("op_1282"), val = tensor([1, 5, 4, 64])]; - tensor var_1283 = reshape(shape = var_1282, x = var_1281)[name = tensor("op_1283")]; + tensor var_851 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_852 = const()[name = tensor("op_852"), val = tensor([1, 5, 4, 64])]; + tensor var_853 = reshape(shape = var_852, x = var_851)[name = tensor("op_853")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_177 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_29)[name = tensor("linear_24")]; + tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; tensor s_t_5 = add(x = prev_scale_5, y = encoder__t_index)[name = tensor("s_t_5")]; tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_1277)[name = tensor("transpose_57")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_1269)[name = tensor("transpose_58")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_847)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_839)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_1293 = const()[name = tensor("op_1293"), val = tensor([5, 1])]; - tensor var_1294 = reshape(shape = var_1293, x = sqrt_s_t_5)[name = tensor("op_1294")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_1294)[name = tensor("M_5")]; - tensor var_1296 = mul(x = qk_5, y = M_5)[name = tensor("op_1296")]; + tensor var_863 = const()[name = tensor("op_863"), val = tensor([5, 1])]; + tensor var_864 = reshape(shape = var_863, x = sqrt_s_t_5)[name = tensor("op_864")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_864)[name = tensor("M_5")]; + tensor var_866 = mul(x = qk_5, y = M_5)[name = tensor("op_866")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_1283)[name = tensor("transpose_56")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_1296, y = v_5)[name = tensor("inner_5")]; - tensor var_1298_transpose_x_0 = const()[name = tensor("op_1298_transpose_x_0"), val = tensor(false)]; - tensor var_1298_transpose_y_0 = const()[name = tensor("op_1298_transpose_y_0"), val = tensor(false)]; - tensor var_1298 = matmul(transpose_x = var_1298_transpose_x_0, transpose_y = var_1298_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_1298")]; - tensor var_1299 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_1299")]; - tensor var_1300 = const()[name = tensor("op_1300"), val = tensor([1, 1, 5, 1])]; - tensor var_1301 = reshape(shape = var_1300, x = var_1299)[name = tensor("op_1301")]; - tensor cross_5 = mul(x = var_1298, y = var_1301)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_853)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_866, y = v_5)[name = tensor("inner_5")]; + tensor var_868_transpose_x_0 = const()[name = tensor("op_868_transpose_x_0"), val = tensor(false)]; + tensor var_868_transpose_y_0 = const()[name = tensor("op_868_transpose_y_0"), val = tensor(false)]; + tensor var_868 = matmul(transpose_x = var_868_transpose_x_0, transpose_y = var_868_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_868")]; + tensor var_869 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_869")]; + tensor var_870 = const()[name = tensor("op_870"), val = tensor([1, 1, 5, 1])]; + tensor var_871 = reshape(shape = var_870, x = var_869)[name = tensor("op_871")]; + tensor cross_5 = mul(x = var_868, y = var_871)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_1304 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_1304")]; - tensor var_1306_transpose_x_1 = const()[name = tensor("op_1306_transpose_x_1"), val = tensor(true)]; - tensor var_1306_transpose_y_1 = const()[name = tensor("op_1306_transpose_y_1"), val = tensor(false)]; - tensor var_1306 = matmul(transpose_x = var_1306_transpose_x_1, transpose_y = var_1306_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_1306")]; - tensor new_kv_unnorm_5 = add(x = var_1304, y = var_1306)[name = tensor("new_kv_unnorm_5")]; - tensor var_1308 = const()[name = tensor("op_1308"), val = tensor(0x1.4p+2)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_1308)[name = tensor("new_scale_5")]; - tensor var_1310 = sqrt(x = new_scale_5)[name = tensor("op_1310")]; - tensor var_1311 = real_div(x = new_kv_unnorm_5, y = var_1310)[name = tensor("op_1311")]; - tensor var_1312_perm_0 = const()[name = tensor("op_1312_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_874 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_874")]; + tensor var_876_transpose_x_1 = const()[name = tensor("op_876_transpose_x_1"), val = tensor(true)]; + tensor var_876_transpose_y_1 = const()[name = tensor("op_876_transpose_y_1"), val = tensor(false)]; + tensor var_876 = matmul(transpose_x = var_876_transpose_x_1, transpose_y = var_876_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_876")]; + tensor new_kv_unnorm_5 = add(x = var_874, y = var_876)[name = tensor("new_kv_unnorm_5")]; + tensor var_878 = const()[name = tensor("op_878"), val = tensor(0x1.4p+2)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_878)[name = tensor("new_scale_5")]; + tensor var_880 = sqrt(x = new_scale_5)[name = tensor("op_880")]; + tensor var_881 = real_div(x = new_kv_unnorm_5, y = var_880)[name = tensor("op_881")]; + tensor var_882_perm_0 = const()[name = tensor("op_882_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_1312 = transpose(perm = var_1312_perm_0, x = out_13)[name = tensor("transpose_55")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_25, x = var_1312)[name = tensor("out_15")]; - tensor var_1316 = const()[name = tensor("op_1316"), val = tensor([1, 5, 256])]; - tensor out_17 = reshape(shape = var_1316, x = out_15)[name = tensor("out_17")]; - tensor var_1318 = silu(x = input_177)[name = tensor("op_1318")]; - tensor input_179 = mul(x = var_1318, y = out_17)[name = tensor("input_179")]; - tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_179)[name = tensor("linear_25")]; - tensor x_31 = add(x = input_173, y = ret_out_5)[name = tensor("x_31")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor var_882 = transpose(perm = var_882_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_25, x = var_882)[name = tensor("out_15")]; + tensor var_886 = const()[name = tensor("op_886"), val = tensor([1, 5, 256])]; + tensor out_17 = reshape(shape = var_886, x = out_15)[name = tensor("out_17")]; + tensor var_888 = silu(x = input_97)[name = tensor("op_888")]; + tensor input_99 = mul(x = var_888, y = out_17)[name = tensor("input_99")]; + tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; + tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; + tensor window_25_begin_0 = const()[name = tensor("window_25_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_25_end_0 = const()[name = tensor("window_25_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_25_end_mask_0 = const()[name = tensor("window_25_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_25_squeeze_mask_0 = const()[name = tensor("window_25_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_25 = slice_by_index(begin = window_25_begin_0, end = window_25_end_0, end_mask = window_25_end_mask_0, squeeze_mask = window_25_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_25")]; tensor filled_21_begin_0 = const()[name = tensor("filled_21_begin_0"), val = tensor([2, 0])]; tensor filled_21_end_0 = const()[name = tensor("filled_21_end_0"), val = tensor([3, 1])]; tensor filled_21_end_mask_0 = const()[name = tensor("filled_21_end_mask_0"), val = tensor([false, true])]; tensor filled_21_squeeze_mask_0 = const()[name = tensor("filled_21_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_21 = slice_by_index(begin = filled_21_begin_0, end = filled_21_end_0, end_mask = filled_21_end_mask_0, squeeze_mask = filled_21_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_21")]; - tensor var_1327_begin_0 = const()[name = tensor("op_1327_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1327_end_0 = const()[name = tensor("op_1327_end_0"), val = tensor([1, 1, 256])]; - tensor var_1327_end_mask_0 = const()[name = tensor("op_1327_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1327 = slice_by_index(begin = var_1327_begin_0, end = var_1327_end_0, end_mask = var_1327_end_mask_0, x = x_31)[name = tensor("op_1327")]; - tensor var_1330_begin_0 = const()[name = tensor("op_1330_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1330_end_0 = const()[name = tensor("op_1330_end_0"), val = tensor([1, 16, 256])]; - tensor var_1330_end_mask_0 = const()[name = tensor("op_1330_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1330 = slice_by_index(begin = var_1330_begin_0, end = var_1330_end_0, end_mask = var_1330_end_mask_0, x = window_5)[name = tensor("op_1330")]; + tensor var_897_begin_0 = const()[name = tensor("op_897_begin_0"), val = tensor([0, 0, 0])]; + tensor var_897_end_0 = const()[name = tensor("op_897_end_0"), val = tensor([1, 1, 256])]; + tensor var_897_end_mask_0 = const()[name = tensor("op_897_end_mask_0"), val = tensor([true, false, true])]; + tensor var_897 = slice_by_index(begin = var_897_begin_0, end = var_897_end_0, end_mask = var_897_end_mask_0, x = x_15)[name = tensor("op_897")]; + tensor var_900_begin_0 = const()[name = tensor("op_900_begin_0"), val = tensor([0, 1, 0])]; + tensor var_900_end_0 = const()[name = tensor("op_900_end_0"), val = tensor([1, 16, 256])]; + tensor var_900_end_mask_0 = const()[name = tensor("op_900_end_mask_0"), val = tensor([true, true, true])]; + tensor var_900 = slice_by_index(begin = var_900_begin_0, end = var_900_end_0, end_mask = var_900_end_mask_0, x = window_25)[name = tensor("op_900")]; tensor rolled_21_interleave_0 = const()[name = tensor("rolled_21_interleave_0"), val = tensor(false)]; - tensor rolled_21 = concat(axis = var_34, interleave = rolled_21_interleave_0, values = (var_1330, var_1327))[name = tensor("rolled_21")]; + tensor rolled_21 = concat(axis = var_34, interleave = rolled_21_interleave_0, values = (var_900, var_897))[name = tensor("rolled_21")]; tensor clip_20 = clip(alpha = var_22, beta = var_21, x = filled_21)[name = tensor("clip_20")]; - tensor var_1335 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_1335")]; - tensor var_1336_dtype_0 = const()[name = tensor("op_1336_dtype_0"), val = tensor("fp32")]; - tensor var_1337 = const()[name = tensor("op_1337"), val = tensor([1, 16, 1])]; - tensor var_1336 = cast(dtype = var_1336_dtype_0, x = var_1335)[name = tensor("cast_171")]; - tensor fill_mask_21 = reshape(shape = var_1337, x = var_1336)[name = tensor("fill_mask_21")]; + tensor var_905 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_905")]; + tensor var_906_dtype_0 = const()[name = tensor("op_906_dtype_0"), val = tensor("fp32")]; + tensor var_907 = const()[name = tensor("op_907"), val = tensor([1, 16, 1])]; + tensor var_906 = cast(dtype = var_906_dtype_0, x = var_905)[name = tensor("cast_171")]; + tensor fill_mask_21 = reshape(shape = var_907, x = var_906)[name = tensor("fill_mask_21")]; tensor x_expanded_21_reps_0 = const()[name = tensor("x_expanded_21_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_1327)[name = tensor("x_expanded_21")]; - tensor var_1341 = sub(x = var_18, y = fill_mask_21)[name = tensor("op_1341")]; - tensor var_1342 = mul(x = window_5, y = var_1341)[name = tensor("op_1342")]; - tensor var_1343 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_1343")]; - tensor filled_window_21 = add(x = var_1342, y = var_1343)[name = tensor("filled_window_21")]; - tensor var_1345 = greater_equal(x = filled_21, y = var_17)[name = tensor("op_1345")]; - tensor var_1346_dtype_0 = const()[name = tensor("op_1346_dtype_0"), val = tensor("fp32")]; - tensor var_1347 = const()[name = tensor("op_1347"), val = tensor([1, 1, 1])]; - tensor var_1346 = cast(dtype = var_1346_dtype_0, x = var_1345)[name = tensor("cast_170")]; - tensor use_roll_21 = reshape(shape = var_1347, x = var_1346)[name = tensor("use_roll_21")]; - tensor var_1349 = sub(x = var_18, y = use_roll_21)[name = tensor("op_1349")]; - tensor var_1350 = mul(x = filled_window_21, y = var_1349)[name = tensor("op_1350")]; - tensor var_1351 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_1351")]; - tensor input_181 = add(x = var_1350, y = var_1351)[name = tensor("input_181")]; - tensor var_1353 = const()[name = tensor("op_1353"), val = tensor(0x1p+0)]; - tensor var_1354 = add(x = filled_21, y = var_1353)[name = tensor("op_1354")]; - tensor clip_21 = clip(alpha = var_22, beta = var_17, x = var_1354)[name = tensor("clip_21")]; - tensor x_33_axes_0 = const()[name = tensor("x_33_axes_0"), val = tensor([-1])]; - tensor x_33 = layer_norm(axes = x_33_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_181)[name = tensor("x_33")]; - tensor input_183_perm_0 = const()[name = tensor("input_183_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_71_pad_type_0 = const()[name = tensor("inputs_71_pad_type_0"), val = tensor("valid")]; - tensor inputs_71_strides_0 = const()[name = tensor("inputs_71_strides_0"), val = tensor([1])]; - tensor inputs_71_pad_0 = const()[name = tensor("inputs_71_pad_0"), val = tensor([0, 0])]; - tensor inputs_71_dilations_0 = const()[name = tensor("inputs_71_dilations_0"), val = tensor([1])]; - tensor inputs_71_groups_0 = const()[name = tensor("inputs_71_groups_0"), val = tensor(1)]; - tensor input_183 = transpose(perm = input_183_perm_0, x = x_33)[name = tensor("transpose_54")]; - tensor inputs_71 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_71_dilations_0, groups = inputs_71_groups_0, pad = inputs_71_pad_0, pad_type = inputs_71_pad_type_0, strides = inputs_71_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_183)[name = tensor("inputs_71")]; - tensor var_1375_split_sizes_0 = const()[name = tensor("op_1375_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1375_axis_0 = const()[name = tensor("op_1375_axis_0"), val = tensor(1)]; - tensor var_1375_0, tensor var_1375_1 = split(axis = var_1375_axis_0, split_sizes = var_1375_split_sizes_0, x = inputs_71)[name = tensor("op_1375")]; - tensor var_1377 = sigmoid(x = var_1375_1)[name = tensor("op_1377")]; - tensor inputs_73 = mul(x = var_1375_0, y = var_1377)[name = tensor("inputs_73")]; - tensor outputs_aug_21_pad_type_0 = const()[name = tensor("outputs_aug_21_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_21_pad_0 = const()[name = tensor("outputs_aug_21_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_21_groups_0 = const()[name = tensor("outputs_aug_21_groups_0"), val = tensor(256)]; - tensor outputs_aug_21_strides_0 = const()[name = tensor("outputs_aug_21_strides_0"), val = tensor([1])]; - tensor outputs_aug_21_dilations_0 = const()[name = tensor("outputs_aug_21_dilations_0"), val = tensor([1])]; - tensor outputs_aug_21 = conv(dilations = outputs_aug_21_dilations_0, groups = outputs_aug_21_groups_0, pad = outputs_aug_21_pad_0, pad_type = outputs_aug_21_pad_type_0, strides = outputs_aug_21_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_73)[name = tensor("outputs_aug_21")]; - tensor input_185_begin_0 = const()[name = tensor("input_185_begin_0"), val = tensor([0, 0, 0])]; - tensor input_185_end_0 = const()[name = tensor("input_185_end_0"), val = tensor([1, 256, 16])]; - tensor input_185_end_mask_0 = const()[name = tensor("input_185_end_mask_0"), val = tensor([true, true, false])]; - tensor input_185 = slice_by_index(begin = input_185_begin_0, end = input_185_end_0, end_mask = input_185_end_mask_0, x = outputs_aug_21)[name = tensor("input_185")]; - tensor inputs_75 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_185)[name = tensor("inputs_75")]; - tensor input_187 = silu(x = inputs_75)[name = tensor("input_187")]; - tensor input_189_pad_type_0 = const()[name = tensor("input_189_pad_type_0"), val = tensor("valid")]; - tensor input_189_strides_0 = const()[name = tensor("input_189_strides_0"), val = tensor([1])]; - tensor input_189_pad_0 = const()[name = tensor("input_189_pad_0"), val = tensor([0, 0])]; - tensor input_189_dilations_0 = const()[name = tensor("input_189_dilations_0"), val = tensor([1])]; - tensor input_189_groups_0 = const()[name = tensor("input_189_groups_0"), val = tensor(1)]; - tensor input_189 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_189_dilations_0, groups = input_189_groups_0, pad = input_189_pad_0, pad_type = input_189_pad_type_0, strides = input_189_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_187)[name = tensor("input_189")]; - tensor conv_out_21_perm_0 = const()[name = tensor("conv_out_21_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1408_begin_0 = const()[name = tensor("op_1408_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1408_end_0 = const()[name = tensor("op_1408_end_0"), val = tensor([1, 16, 256])]; - tensor var_1408_end_mask_0 = const()[name = tensor("op_1408_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_21 = transpose(perm = conv_out_21_perm_0, x = input_189)[name = tensor("transpose_53")]; - tensor var_1408 = slice_by_index(begin = var_1408_begin_0, end = var_1408_end_0, end_mask = var_1408_end_mask_0, x = conv_out_21)[name = tensor("op_1408")]; - tensor var_1411_begin_0 = const()[name = tensor("op_1411_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1411_end_0 = const()[name = tensor("op_1411_end_0"), val = tensor([1, 2, 256])]; - tensor var_1411_end_mask_0 = const()[name = tensor("op_1411_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1411 = slice_by_index(begin = var_1411_begin_0, end = var_1411_end_0, end_mask = var_1411_end_mask_0, x = x_31)[name = tensor("op_1411")]; - tensor var_1414_begin_0 = const()[name = tensor("op_1414_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1414_end_0 = const()[name = tensor("op_1414_end_0"), val = tensor([1, 16, 256])]; - tensor var_1414_end_mask_0 = const()[name = tensor("op_1414_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1414 = slice_by_index(begin = var_1414_begin_0, end = var_1414_end_0, end_mask = var_1414_end_mask_0, x = input_181)[name = tensor("op_1414")]; + tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_897)[name = tensor("x_expanded_21")]; + tensor var_911 = sub(x = var_18, y = fill_mask_21)[name = tensor("op_911")]; + tensor var_912 = mul(x = window_25, y = var_911)[name = tensor("op_912")]; + tensor var_913 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_913")]; + tensor filled_window_21 = add(x = var_912, y = var_913)[name = tensor("filled_window_21")]; + tensor var_915 = greater_equal(x = filled_21, y = var_17)[name = tensor("op_915")]; + tensor var_916_dtype_0 = const()[name = tensor("op_916_dtype_0"), val = tensor("fp32")]; + tensor var_917 = const()[name = tensor("op_917"), val = tensor([1, 1, 1])]; + tensor var_916 = cast(dtype = var_916_dtype_0, x = var_915)[name = tensor("cast_170")]; + tensor use_roll_21 = reshape(shape = var_917, x = var_916)[name = tensor("use_roll_21")]; + tensor var_919 = sub(x = var_18, y = use_roll_21)[name = tensor("op_919")]; + tensor var_920 = mul(x = filled_window_21, y = var_919)[name = tensor("op_920")]; + tensor var_921 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_921")]; + tensor window_27 = add(x = var_920, y = var_921)[name = tensor("window_27")]; + tensor var_923 = const()[name = tensor("op_923"), val = tensor(0x1p+0)]; + tensor var_924 = add(x = filled_21, y = var_923)[name = tensor("op_924")]; + tensor clip_21 = clip(alpha = var_22, beta = var_17, x = var_924)[name = tensor("clip_21")]; + tensor var_927_begin_0 = const()[name = tensor("op_927_begin_0"), val = tensor([0, 1, 0])]; + tensor var_927_end_0 = const()[name = tensor("op_927_end_0"), val = tensor([1, 2, 256])]; + tensor var_927_end_mask_0 = const()[name = tensor("op_927_end_mask_0"), val = tensor([true, false, true])]; + tensor var_927 = slice_by_index(begin = var_927_begin_0, end = var_927_end_0, end_mask = var_927_end_mask_0, x = x_15)[name = tensor("op_927")]; + tensor var_930_begin_0 = const()[name = tensor("op_930_begin_0"), val = tensor([0, 1, 0])]; + tensor var_930_end_0 = const()[name = tensor("op_930_end_0"), val = tensor([1, 16, 256])]; + tensor var_930_end_mask_0 = const()[name = tensor("op_930_end_mask_0"), val = tensor([true, true, true])]; + tensor var_930 = slice_by_index(begin = var_930_begin_0, end = var_930_end_0, end_mask = var_930_end_mask_0, x = window_27)[name = tensor("op_930")]; tensor rolled_23_interleave_0 = const()[name = tensor("rolled_23_interleave_0"), val = tensor(false)]; - tensor rolled_23 = concat(axis = var_34, interleave = rolled_23_interleave_0, values = (var_1414, var_1411))[name = tensor("rolled_23")]; + tensor rolled_23 = concat(axis = var_34, interleave = rolled_23_interleave_0, values = (var_930, var_927))[name = tensor("rolled_23")]; tensor clip_22 = clip(alpha = var_22, beta = var_21, x = clip_21)[name = tensor("clip_22")]; - tensor var_1419 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_1419")]; - tensor var_1420_dtype_0 = const()[name = tensor("op_1420_dtype_0"), val = tensor("fp32")]; - tensor var_1421 = const()[name = tensor("op_1421"), val = tensor([1, 16, 1])]; - tensor var_1420 = cast(dtype = var_1420_dtype_0, x = var_1419)[name = tensor("cast_169")]; - tensor fill_mask_23 = reshape(shape = var_1421, x = var_1420)[name = tensor("fill_mask_23")]; + tensor var_935 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_935")]; + tensor var_936_dtype_0 = const()[name = tensor("op_936_dtype_0"), val = tensor("fp32")]; + tensor var_937 = const()[name = tensor("op_937"), val = tensor([1, 16, 1])]; + tensor var_936 = cast(dtype = var_936_dtype_0, x = var_935)[name = tensor("cast_169")]; + tensor fill_mask_23 = reshape(shape = var_937, x = var_936)[name = tensor("fill_mask_23")]; tensor x_expanded_23_reps_0 = const()[name = tensor("x_expanded_23_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_23 = tile(reps = x_expanded_23_reps_0, x = var_1411)[name = tensor("x_expanded_23")]; - tensor var_1425 = sub(x = var_18, y = fill_mask_23)[name = tensor("op_1425")]; - tensor var_1426 = mul(x = input_181, y = var_1425)[name = tensor("op_1426")]; - tensor var_1427 = mul(x = x_expanded_23, y = fill_mask_23)[name = tensor("op_1427")]; - tensor filled_window_23 = add(x = var_1426, y = var_1427)[name = tensor("filled_window_23")]; - tensor var_1429 = greater_equal(x = clip_21, y = var_17)[name = tensor("op_1429")]; - tensor var_1430_dtype_0 = const()[name = tensor("op_1430_dtype_0"), val = tensor("fp32")]; - tensor var_1431 = const()[name = tensor("op_1431"), val = tensor([1, 1, 1])]; - tensor var_1430 = cast(dtype = var_1430_dtype_0, x = var_1429)[name = tensor("cast_168")]; - tensor use_roll_23 = reshape(shape = var_1431, x = var_1430)[name = tensor("use_roll_23")]; - tensor var_1433 = sub(x = var_18, y = use_roll_23)[name = tensor("op_1433")]; - tensor var_1434 = mul(x = filled_window_23, y = var_1433)[name = tensor("op_1434")]; - tensor var_1435 = mul(x = rolled_23, y = use_roll_23)[name = tensor("op_1435")]; - tensor input_191 = add(x = var_1434, y = var_1435)[name = tensor("input_191")]; - tensor var_1437 = const()[name = tensor("op_1437"), val = tensor(0x1p+0)]; - tensor var_1438 = add(x = clip_21, y = var_1437)[name = tensor("op_1438")]; - tensor clip_23 = clip(alpha = var_22, beta = var_17, x = var_1438)[name = tensor("clip_23")]; - tensor x_35_axes_0 = const()[name = tensor("x_35_axes_0"), val = tensor([-1])]; - tensor x_35 = layer_norm(axes = x_35_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_191)[name = tensor("x_35")]; - tensor input_193_perm_0 = const()[name = tensor("input_193_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_77_pad_type_0 = const()[name = tensor("inputs_77_pad_type_0"), val = tensor("valid")]; - tensor inputs_77_strides_0 = const()[name = tensor("inputs_77_strides_0"), val = tensor([1])]; - tensor inputs_77_pad_0 = const()[name = tensor("inputs_77_pad_0"), val = tensor([0, 0])]; - tensor inputs_77_dilations_0 = const()[name = tensor("inputs_77_dilations_0"), val = tensor([1])]; - tensor inputs_77_groups_0 = const()[name = tensor("inputs_77_groups_0"), val = tensor(1)]; - tensor input_193 = transpose(perm = input_193_perm_0, x = x_35)[name = tensor("transpose_52")]; - tensor inputs_77 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_77_dilations_0, groups = inputs_77_groups_0, pad = inputs_77_pad_0, pad_type = inputs_77_pad_type_0, strides = inputs_77_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_193)[name = tensor("inputs_77")]; - tensor var_1459_split_sizes_0 = const()[name = tensor("op_1459_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1459_axis_0 = const()[name = tensor("op_1459_axis_0"), val = tensor(1)]; - tensor var_1459_0, tensor var_1459_1 = split(axis = var_1459_axis_0, split_sizes = var_1459_split_sizes_0, x = inputs_77)[name = tensor("op_1459")]; - tensor var_1461 = sigmoid(x = var_1459_1)[name = tensor("op_1461")]; - tensor inputs_79 = mul(x = var_1459_0, y = var_1461)[name = tensor("inputs_79")]; - tensor outputs_aug_23_pad_type_0 = const()[name = tensor("outputs_aug_23_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_23_pad_0 = const()[name = tensor("outputs_aug_23_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_23_groups_0 = const()[name = tensor("outputs_aug_23_groups_0"), val = tensor(256)]; - tensor outputs_aug_23_strides_0 = const()[name = tensor("outputs_aug_23_strides_0"), val = tensor([1])]; - tensor outputs_aug_23_dilations_0 = const()[name = tensor("outputs_aug_23_dilations_0"), val = tensor([1])]; - tensor outputs_aug_23 = conv(dilations = outputs_aug_23_dilations_0, groups = outputs_aug_23_groups_0, pad = outputs_aug_23_pad_0, pad_type = outputs_aug_23_pad_type_0, strides = outputs_aug_23_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_79)[name = tensor("outputs_aug_23")]; - tensor input_195_begin_0 = const()[name = tensor("input_195_begin_0"), val = tensor([0, 0, 0])]; - tensor input_195_end_0 = const()[name = tensor("input_195_end_0"), val = tensor([1, 256, 16])]; - tensor input_195_end_mask_0 = const()[name = tensor("input_195_end_mask_0"), val = tensor([true, true, false])]; - tensor input_195 = slice_by_index(begin = input_195_begin_0, end = input_195_end_0, end_mask = input_195_end_mask_0, x = outputs_aug_23)[name = tensor("input_195")]; - tensor inputs_81 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_195)[name = tensor("inputs_81")]; - tensor input_197 = silu(x = inputs_81)[name = tensor("input_197")]; - tensor input_199_pad_type_0 = const()[name = tensor("input_199_pad_type_0"), val = tensor("valid")]; - tensor input_199_strides_0 = const()[name = tensor("input_199_strides_0"), val = tensor([1])]; - tensor input_199_pad_0 = const()[name = tensor("input_199_pad_0"), val = tensor([0, 0])]; - tensor input_199_dilations_0 = const()[name = tensor("input_199_dilations_0"), val = tensor([1])]; - tensor input_199_groups_0 = const()[name = tensor("input_199_groups_0"), val = tensor(1)]; - tensor input_199 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_199_dilations_0, groups = input_199_groups_0, pad = input_199_pad_0, pad_type = input_199_pad_type_0, strides = input_199_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_197)[name = tensor("input_199")]; - tensor conv_out_23_perm_0 = const()[name = tensor("conv_out_23_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1492_begin_0 = const()[name = tensor("op_1492_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1492_end_0 = const()[name = tensor("op_1492_end_0"), val = tensor([1, 16, 256])]; - tensor var_1492_end_mask_0 = const()[name = tensor("op_1492_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_23 = transpose(perm = conv_out_23_perm_0, x = input_199)[name = tensor("transpose_51")]; - tensor var_1492 = slice_by_index(begin = var_1492_begin_0, end = var_1492_end_0, end_mask = var_1492_end_mask_0, x = conv_out_23)[name = tensor("op_1492")]; - tensor var_1495_begin_0 = const()[name = tensor("op_1495_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1495_end_0 = const()[name = tensor("op_1495_end_0"), val = tensor([1, 3, 256])]; - tensor var_1495_end_mask_0 = const()[name = tensor("op_1495_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1495 = slice_by_index(begin = var_1495_begin_0, end = var_1495_end_0, end_mask = var_1495_end_mask_0, x = x_31)[name = tensor("op_1495")]; - tensor var_1498_begin_0 = const()[name = tensor("op_1498_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1498_end_0 = const()[name = tensor("op_1498_end_0"), val = tensor([1, 16, 256])]; - tensor var_1498_end_mask_0 = const()[name = tensor("op_1498_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1498 = slice_by_index(begin = var_1498_begin_0, end = var_1498_end_0, end_mask = var_1498_end_mask_0, x = input_191)[name = tensor("op_1498")]; + tensor x_expanded_23 = tile(reps = x_expanded_23_reps_0, x = var_927)[name = tensor("x_expanded_23")]; + tensor var_941 = sub(x = var_18, y = fill_mask_23)[name = tensor("op_941")]; + tensor var_942 = mul(x = window_27, y = var_941)[name = tensor("op_942")]; + tensor var_943 = mul(x = x_expanded_23, y = fill_mask_23)[name = tensor("op_943")]; + tensor filled_window_23 = add(x = var_942, y = var_943)[name = tensor("filled_window_23")]; + tensor var_945 = greater_equal(x = clip_21, y = var_17)[name = tensor("op_945")]; + tensor var_946_dtype_0 = const()[name = tensor("op_946_dtype_0"), val = tensor("fp32")]; + tensor var_947 = const()[name = tensor("op_947"), val = tensor([1, 1, 1])]; + tensor var_946 = cast(dtype = var_946_dtype_0, x = var_945)[name = tensor("cast_168")]; + tensor use_roll_23 = reshape(shape = var_947, x = var_946)[name = tensor("use_roll_23")]; + tensor var_949 = sub(x = var_18, y = use_roll_23)[name = tensor("op_949")]; + tensor var_950 = mul(x = filled_window_23, y = var_949)[name = tensor("op_950")]; + tensor var_951 = mul(x = rolled_23, y = use_roll_23)[name = tensor("op_951")]; + tensor window_29 = add(x = var_950, y = var_951)[name = tensor("window_29")]; + tensor var_953 = const()[name = tensor("op_953"), val = tensor(0x1p+0)]; + tensor var_954 = add(x = clip_21, y = var_953)[name = tensor("op_954")]; + tensor clip_23 = clip(alpha = var_22, beta = var_17, x = var_954)[name = tensor("clip_23")]; + tensor var_957_begin_0 = const()[name = tensor("op_957_begin_0"), val = tensor([0, 2, 0])]; + tensor var_957_end_0 = const()[name = tensor("op_957_end_0"), val = tensor([1, 3, 256])]; + tensor var_957_end_mask_0 = const()[name = tensor("op_957_end_mask_0"), val = tensor([true, false, true])]; + tensor var_957 = slice_by_index(begin = var_957_begin_0, end = var_957_end_0, end_mask = var_957_end_mask_0, x = x_15)[name = tensor("op_957")]; + tensor var_960_begin_0 = const()[name = tensor("op_960_begin_0"), val = tensor([0, 1, 0])]; + tensor var_960_end_0 = const()[name = tensor("op_960_end_0"), val = tensor([1, 16, 256])]; + tensor var_960_end_mask_0 = const()[name = tensor("op_960_end_mask_0"), val = tensor([true, true, true])]; + tensor var_960 = slice_by_index(begin = var_960_begin_0, end = var_960_end_0, end_mask = var_960_end_mask_0, x = window_29)[name = tensor("op_960")]; tensor rolled_25_interleave_0 = const()[name = tensor("rolled_25_interleave_0"), val = tensor(false)]; - tensor rolled_25 = concat(axis = var_34, interleave = rolled_25_interleave_0, values = (var_1498, var_1495))[name = tensor("rolled_25")]; + tensor rolled_25 = concat(axis = var_34, interleave = rolled_25_interleave_0, values = (var_960, var_957))[name = tensor("rolled_25")]; tensor clip_24 = clip(alpha = var_22, beta = var_21, x = clip_23)[name = tensor("clip_24")]; - tensor var_1503 = equal(x = encoder__conv_positions, y = clip_24)[name = tensor("op_1503")]; - tensor var_1504_dtype_0 = const()[name = tensor("op_1504_dtype_0"), val = tensor("fp32")]; - tensor var_1505 = const()[name = tensor("op_1505"), val = tensor([1, 16, 1])]; - tensor var_1504 = cast(dtype = var_1504_dtype_0, x = var_1503)[name = tensor("cast_167")]; - tensor fill_mask_25 = reshape(shape = var_1505, x = var_1504)[name = tensor("fill_mask_25")]; + tensor var_965 = equal(x = encoder__conv_positions, y = clip_24)[name = tensor("op_965")]; + tensor var_966_dtype_0 = const()[name = tensor("op_966_dtype_0"), val = tensor("fp32")]; + tensor var_967 = const()[name = tensor("op_967"), val = tensor([1, 16, 1])]; + tensor var_966 = cast(dtype = var_966_dtype_0, x = var_965)[name = tensor("cast_167")]; + tensor fill_mask_25 = reshape(shape = var_967, x = var_966)[name = tensor("fill_mask_25")]; tensor x_expanded_25_reps_0 = const()[name = tensor("x_expanded_25_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_25 = tile(reps = x_expanded_25_reps_0, x = var_1495)[name = tensor("x_expanded_25")]; - tensor var_1509 = sub(x = var_18, y = fill_mask_25)[name = tensor("op_1509")]; - tensor var_1510 = mul(x = input_191, y = var_1509)[name = tensor("op_1510")]; - tensor var_1511 = mul(x = x_expanded_25, y = fill_mask_25)[name = tensor("op_1511")]; - tensor filled_window_25 = add(x = var_1510, y = var_1511)[name = tensor("filled_window_25")]; - tensor var_1513 = greater_equal(x = clip_23, y = var_17)[name = tensor("op_1513")]; - tensor var_1514_dtype_0 = const()[name = tensor("op_1514_dtype_0"), val = tensor("fp32")]; - tensor var_1515 = const()[name = tensor("op_1515"), val = tensor([1, 1, 1])]; - tensor var_1514 = cast(dtype = var_1514_dtype_0, x = var_1513)[name = tensor("cast_166")]; - tensor use_roll_25 = reshape(shape = var_1515, x = var_1514)[name = tensor("use_roll_25")]; - tensor var_1517 = sub(x = var_18, y = use_roll_25)[name = tensor("op_1517")]; - tensor var_1518 = mul(x = filled_window_25, y = var_1517)[name = tensor("op_1518")]; - tensor var_1519 = mul(x = rolled_25, y = use_roll_25)[name = tensor("op_1519")]; - tensor input_201 = add(x = var_1518, y = var_1519)[name = tensor("input_201")]; - tensor var_1521 = const()[name = tensor("op_1521"), val = tensor(0x1p+0)]; - tensor var_1522 = add(x = clip_23, y = var_1521)[name = tensor("op_1522")]; - tensor clip_25 = clip(alpha = var_22, beta = var_17, x = var_1522)[name = tensor("clip_25")]; - tensor x_37_axes_0 = const()[name = tensor("x_37_axes_0"), val = tensor([-1])]; - tensor x_37 = layer_norm(axes = x_37_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_201)[name = tensor("x_37")]; - tensor input_203_perm_0 = const()[name = tensor("input_203_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_83_pad_type_0 = const()[name = tensor("inputs_83_pad_type_0"), val = tensor("valid")]; - tensor inputs_83_strides_0 = const()[name = tensor("inputs_83_strides_0"), val = tensor([1])]; - tensor inputs_83_pad_0 = const()[name = tensor("inputs_83_pad_0"), val = tensor([0, 0])]; - tensor inputs_83_dilations_0 = const()[name = tensor("inputs_83_dilations_0"), val = tensor([1])]; - tensor inputs_83_groups_0 = const()[name = tensor("inputs_83_groups_0"), val = tensor(1)]; - tensor input_203 = transpose(perm = input_203_perm_0, x = x_37)[name = tensor("transpose_50")]; - tensor inputs_83 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_83_dilations_0, groups = inputs_83_groups_0, pad = inputs_83_pad_0, pad_type = inputs_83_pad_type_0, strides = inputs_83_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_203)[name = tensor("inputs_83")]; - tensor var_1543_split_sizes_0 = const()[name = tensor("op_1543_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1543_axis_0 = const()[name = tensor("op_1543_axis_0"), val = tensor(1)]; - tensor var_1543_0, tensor var_1543_1 = split(axis = var_1543_axis_0, split_sizes = var_1543_split_sizes_0, x = inputs_83)[name = tensor("op_1543")]; - tensor var_1545 = sigmoid(x = var_1543_1)[name = tensor("op_1545")]; - tensor inputs_85 = mul(x = var_1543_0, y = var_1545)[name = tensor("inputs_85")]; - tensor outputs_aug_25_pad_type_0 = const()[name = tensor("outputs_aug_25_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_25_pad_0 = const()[name = tensor("outputs_aug_25_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_25_groups_0 = const()[name = tensor("outputs_aug_25_groups_0"), val = tensor(256)]; - tensor outputs_aug_25_strides_0 = const()[name = tensor("outputs_aug_25_strides_0"), val = tensor([1])]; - tensor outputs_aug_25_dilations_0 = const()[name = tensor("outputs_aug_25_dilations_0"), val = tensor([1])]; - tensor outputs_aug_25 = conv(dilations = outputs_aug_25_dilations_0, groups = outputs_aug_25_groups_0, pad = outputs_aug_25_pad_0, pad_type = outputs_aug_25_pad_type_0, strides = outputs_aug_25_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_85)[name = tensor("outputs_aug_25")]; - tensor input_205_begin_0 = const()[name = tensor("input_205_begin_0"), val = tensor([0, 0, 0])]; - tensor input_205_end_0 = const()[name = tensor("input_205_end_0"), val = tensor([1, 256, 16])]; - tensor input_205_end_mask_0 = const()[name = tensor("input_205_end_mask_0"), val = tensor([true, true, false])]; - tensor input_205 = slice_by_index(begin = input_205_begin_0, end = input_205_end_0, end_mask = input_205_end_mask_0, x = outputs_aug_25)[name = tensor("input_205")]; - tensor inputs_87 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_205)[name = tensor("inputs_87")]; - tensor input_207 = silu(x = inputs_87)[name = tensor("input_207")]; - tensor input_209_pad_type_0 = const()[name = tensor("input_209_pad_type_0"), val = tensor("valid")]; - tensor input_209_strides_0 = const()[name = tensor("input_209_strides_0"), val = tensor([1])]; - tensor input_209_pad_0 = const()[name = tensor("input_209_pad_0"), val = tensor([0, 0])]; - tensor input_209_dilations_0 = const()[name = tensor("input_209_dilations_0"), val = tensor([1])]; - tensor input_209_groups_0 = const()[name = tensor("input_209_groups_0"), val = tensor(1)]; - tensor input_209 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_209_dilations_0, groups = input_209_groups_0, pad = input_209_pad_0, pad_type = input_209_pad_type_0, strides = input_209_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_207)[name = tensor("input_209")]; - tensor conv_out_25_perm_0 = const()[name = tensor("conv_out_25_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1576_begin_0 = const()[name = tensor("op_1576_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1576_end_0 = const()[name = tensor("op_1576_end_0"), val = tensor([1, 16, 256])]; - tensor var_1576_end_mask_0 = const()[name = tensor("op_1576_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_25 = transpose(perm = conv_out_25_perm_0, x = input_209)[name = tensor("transpose_49")]; - tensor var_1576 = slice_by_index(begin = var_1576_begin_0, end = var_1576_end_0, end_mask = var_1576_end_mask_0, x = conv_out_25)[name = tensor("op_1576")]; - tensor var_1579_begin_0 = const()[name = tensor("op_1579_begin_0"), val = tensor([0, 3, 0])]; - tensor var_1579_end_0 = const()[name = tensor("op_1579_end_0"), val = tensor([1, 4, 256])]; - tensor var_1579_end_mask_0 = const()[name = tensor("op_1579_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1579 = slice_by_index(begin = var_1579_begin_0, end = var_1579_end_0, end_mask = var_1579_end_mask_0, x = x_31)[name = tensor("op_1579")]; - tensor var_1582_begin_0 = const()[name = tensor("op_1582_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1582_end_0 = const()[name = tensor("op_1582_end_0"), val = tensor([1, 16, 256])]; - tensor var_1582_end_mask_0 = const()[name = tensor("op_1582_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1582 = slice_by_index(begin = var_1582_begin_0, end = var_1582_end_0, end_mask = var_1582_end_mask_0, x = input_201)[name = tensor("op_1582")]; + tensor x_expanded_25 = tile(reps = x_expanded_25_reps_0, x = var_957)[name = tensor("x_expanded_25")]; + tensor var_971 = sub(x = var_18, y = fill_mask_25)[name = tensor("op_971")]; + tensor var_972 = mul(x = window_29, y = var_971)[name = tensor("op_972")]; + tensor var_973 = mul(x = x_expanded_25, y = fill_mask_25)[name = tensor("op_973")]; + tensor filled_window_25 = add(x = var_972, y = var_973)[name = tensor("filled_window_25")]; + tensor var_975 = greater_equal(x = clip_23, y = var_17)[name = tensor("op_975")]; + tensor var_976_dtype_0 = const()[name = tensor("op_976_dtype_0"), val = tensor("fp32")]; + tensor var_977 = const()[name = tensor("op_977"), val = tensor([1, 1, 1])]; + tensor var_976 = cast(dtype = var_976_dtype_0, x = var_975)[name = tensor("cast_166")]; + tensor use_roll_25 = reshape(shape = var_977, x = var_976)[name = tensor("use_roll_25")]; + tensor var_979 = sub(x = var_18, y = use_roll_25)[name = tensor("op_979")]; + tensor var_980 = mul(x = filled_window_25, y = var_979)[name = tensor("op_980")]; + tensor var_981 = mul(x = rolled_25, y = use_roll_25)[name = tensor("op_981")]; + tensor window_31 = add(x = var_980, y = var_981)[name = tensor("window_31")]; + tensor var_983 = const()[name = tensor("op_983"), val = tensor(0x1p+0)]; + tensor var_984 = add(x = clip_23, y = var_983)[name = tensor("op_984")]; + tensor clip_25 = clip(alpha = var_22, beta = var_17, x = var_984)[name = tensor("clip_25")]; + tensor var_987_begin_0 = const()[name = tensor("op_987_begin_0"), val = tensor([0, 3, 0])]; + tensor var_987_end_0 = const()[name = tensor("op_987_end_0"), val = tensor([1, 4, 256])]; + tensor var_987_end_mask_0 = const()[name = tensor("op_987_end_mask_0"), val = tensor([true, false, true])]; + tensor var_987 = slice_by_index(begin = var_987_begin_0, end = var_987_end_0, end_mask = var_987_end_mask_0, x = x_15)[name = tensor("op_987")]; + tensor var_990_begin_0 = const()[name = tensor("op_990_begin_0"), val = tensor([0, 1, 0])]; + tensor var_990_end_0 = const()[name = tensor("op_990_end_0"), val = tensor([1, 16, 256])]; + tensor var_990_end_mask_0 = const()[name = tensor("op_990_end_mask_0"), val = tensor([true, true, true])]; + tensor var_990 = slice_by_index(begin = var_990_begin_0, end = var_990_end_0, end_mask = var_990_end_mask_0, x = window_31)[name = tensor("op_990")]; tensor rolled_27_interleave_0 = const()[name = tensor("rolled_27_interleave_0"), val = tensor(false)]; - tensor rolled_27 = concat(axis = var_34, interleave = rolled_27_interleave_0, values = (var_1582, var_1579))[name = tensor("rolled_27")]; + tensor rolled_27 = concat(axis = var_34, interleave = rolled_27_interleave_0, values = (var_990, var_987))[name = tensor("rolled_27")]; tensor clip_26 = clip(alpha = var_22, beta = var_21, x = clip_25)[name = tensor("clip_26")]; - tensor var_1587 = equal(x = encoder__conv_positions, y = clip_26)[name = tensor("op_1587")]; - tensor var_1588_dtype_0 = const()[name = tensor("op_1588_dtype_0"), val = tensor("fp32")]; - tensor var_1589 = const()[name = tensor("op_1589"), val = tensor([1, 16, 1])]; - tensor var_1588 = cast(dtype = var_1588_dtype_0, x = var_1587)[name = tensor("cast_165")]; - tensor fill_mask_27 = reshape(shape = var_1589, x = var_1588)[name = tensor("fill_mask_27")]; + tensor var_995 = equal(x = encoder__conv_positions, y = clip_26)[name = tensor("op_995")]; + tensor var_996_dtype_0 = const()[name = tensor("op_996_dtype_0"), val = tensor("fp32")]; + tensor var_997 = const()[name = tensor("op_997"), val = tensor([1, 16, 1])]; + tensor var_996 = cast(dtype = var_996_dtype_0, x = var_995)[name = tensor("cast_165")]; + tensor fill_mask_27 = reshape(shape = var_997, x = var_996)[name = tensor("fill_mask_27")]; tensor x_expanded_27_reps_0 = const()[name = tensor("x_expanded_27_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_27 = tile(reps = x_expanded_27_reps_0, x = var_1579)[name = tensor("x_expanded_27")]; - tensor var_1593 = sub(x = var_18, y = fill_mask_27)[name = tensor("op_1593")]; - tensor var_1594 = mul(x = input_201, y = var_1593)[name = tensor("op_1594")]; - tensor var_1595 = mul(x = x_expanded_27, y = fill_mask_27)[name = tensor("op_1595")]; - tensor filled_window_27 = add(x = var_1594, y = var_1595)[name = tensor("filled_window_27")]; - tensor var_1597 = greater_equal(x = clip_25, y = var_17)[name = tensor("op_1597")]; - tensor var_1598_dtype_0 = const()[name = tensor("op_1598_dtype_0"), val = tensor("fp32")]; - tensor var_1599 = const()[name = tensor("op_1599"), val = tensor([1, 1, 1])]; - tensor var_1598 = cast(dtype = var_1598_dtype_0, x = var_1597)[name = tensor("cast_164")]; - tensor use_roll_27 = reshape(shape = var_1599, x = var_1598)[name = tensor("use_roll_27")]; - tensor var_1601 = sub(x = var_18, y = use_roll_27)[name = tensor("op_1601")]; - tensor var_1602 = mul(x = filled_window_27, y = var_1601)[name = tensor("op_1602")]; - tensor var_1603 = mul(x = rolled_27, y = use_roll_27)[name = tensor("op_1603")]; - tensor input_211 = add(x = var_1602, y = var_1603)[name = tensor("input_211")]; - tensor var_1605 = const()[name = tensor("op_1605"), val = tensor(0x1p+0)]; - tensor var_1606 = add(x = clip_25, y = var_1605)[name = tensor("op_1606")]; - tensor clip_27 = clip(alpha = var_22, beta = var_17, x = var_1606)[name = tensor("clip_27")]; - tensor x_39_axes_0 = const()[name = tensor("x_39_axes_0"), val = tensor([-1])]; - tensor x_39 = layer_norm(axes = x_39_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_211)[name = tensor("x_39")]; - tensor input_213_perm_0 = const()[name = tensor("input_213_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_89_pad_type_0 = const()[name = tensor("inputs_89_pad_type_0"), val = tensor("valid")]; - tensor inputs_89_strides_0 = const()[name = tensor("inputs_89_strides_0"), val = tensor([1])]; - tensor inputs_89_pad_0 = const()[name = tensor("inputs_89_pad_0"), val = tensor([0, 0])]; - tensor inputs_89_dilations_0 = const()[name = tensor("inputs_89_dilations_0"), val = tensor([1])]; - tensor inputs_89_groups_0 = const()[name = tensor("inputs_89_groups_0"), val = tensor(1)]; - tensor input_213 = transpose(perm = input_213_perm_0, x = x_39)[name = tensor("transpose_48")]; - tensor inputs_89 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_89_dilations_0, groups = inputs_89_groups_0, pad = inputs_89_pad_0, pad_type = inputs_89_pad_type_0, strides = inputs_89_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_213)[name = tensor("inputs_89")]; - tensor var_1627_split_sizes_0 = const()[name = tensor("op_1627_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1627_axis_0 = const()[name = tensor("op_1627_axis_0"), val = tensor(1)]; - tensor var_1627_0, tensor var_1627_1 = split(axis = var_1627_axis_0, split_sizes = var_1627_split_sizes_0, x = inputs_89)[name = tensor("op_1627")]; - tensor var_1629 = sigmoid(x = var_1627_1)[name = tensor("op_1629")]; - tensor inputs_91 = mul(x = var_1627_0, y = var_1629)[name = tensor("inputs_91")]; - tensor outputs_aug_27_pad_type_0 = const()[name = tensor("outputs_aug_27_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_27_pad_0 = const()[name = tensor("outputs_aug_27_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_27_groups_0 = const()[name = tensor("outputs_aug_27_groups_0"), val = tensor(256)]; - tensor outputs_aug_27_strides_0 = const()[name = tensor("outputs_aug_27_strides_0"), val = tensor([1])]; - tensor outputs_aug_27_dilations_0 = const()[name = tensor("outputs_aug_27_dilations_0"), val = tensor([1])]; - tensor outputs_aug_27 = conv(dilations = outputs_aug_27_dilations_0, groups = outputs_aug_27_groups_0, pad = outputs_aug_27_pad_0, pad_type = outputs_aug_27_pad_type_0, strides = outputs_aug_27_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_91)[name = tensor("outputs_aug_27")]; - tensor input_215_begin_0 = const()[name = tensor("input_215_begin_0"), val = tensor([0, 0, 0])]; - tensor input_215_end_0 = const()[name = tensor("input_215_end_0"), val = tensor([1, 256, 16])]; - tensor input_215_end_mask_0 = const()[name = tensor("input_215_end_mask_0"), val = tensor([true, true, false])]; - tensor input_215 = slice_by_index(begin = input_215_begin_0, end = input_215_end_0, end_mask = input_215_end_mask_0, x = outputs_aug_27)[name = tensor("input_215")]; - tensor inputs_93 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_215)[name = tensor("inputs_93")]; - tensor input_217 = silu(x = inputs_93)[name = tensor("input_217")]; - tensor input_219_pad_type_0 = const()[name = tensor("input_219_pad_type_0"), val = tensor("valid")]; - tensor input_219_strides_0 = const()[name = tensor("input_219_strides_0"), val = tensor([1])]; - tensor input_219_pad_0 = const()[name = tensor("input_219_pad_0"), val = tensor([0, 0])]; - tensor input_219_dilations_0 = const()[name = tensor("input_219_dilations_0"), val = tensor([1])]; - tensor input_219_groups_0 = const()[name = tensor("input_219_groups_0"), val = tensor(1)]; - tensor input_219 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_219_dilations_0, groups = input_219_groups_0, pad = input_219_pad_0, pad_type = input_219_pad_type_0, strides = input_219_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_217)[name = tensor("input_219")]; - tensor conv_out_27_perm_0 = const()[name = tensor("conv_out_27_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1660_begin_0 = const()[name = tensor("op_1660_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1660_end_0 = const()[name = tensor("op_1660_end_0"), val = tensor([1, 16, 256])]; - tensor var_1660_end_mask_0 = const()[name = tensor("op_1660_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_27 = transpose(perm = conv_out_27_perm_0, x = input_219)[name = tensor("transpose_47")]; - tensor var_1660 = slice_by_index(begin = var_1660_begin_0, end = var_1660_end_0, end_mask = var_1660_end_mask_0, x = conv_out_27)[name = tensor("op_1660")]; - tensor var_1663_begin_0 = const()[name = tensor("op_1663_begin_0"), val = tensor([0, 4, 0])]; - tensor var_1663_end_0 = const()[name = tensor("op_1663_end_0"), val = tensor([1, 1, 256])]; - tensor var_1663_end_mask_0 = const()[name = tensor("op_1663_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1663 = slice_by_index(begin = var_1663_begin_0, end = var_1663_end_0, end_mask = var_1663_end_mask_0, x = x_31)[name = tensor("op_1663")]; - tensor var_1666_begin_0 = const()[name = tensor("op_1666_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1666_end_0 = const()[name = tensor("op_1666_end_0"), val = tensor([1, 16, 256])]; - tensor var_1666_end_mask_0 = const()[name = tensor("op_1666_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1666 = slice_by_index(begin = var_1666_begin_0, end = var_1666_end_0, end_mask = var_1666_end_mask_0, x = input_211)[name = tensor("op_1666")]; + tensor x_expanded_27 = tile(reps = x_expanded_27_reps_0, x = var_987)[name = tensor("x_expanded_27")]; + tensor var_1001 = sub(x = var_18, y = fill_mask_27)[name = tensor("op_1001")]; + tensor var_1002 = mul(x = window_31, y = var_1001)[name = tensor("op_1002")]; + tensor var_1003 = mul(x = x_expanded_27, y = fill_mask_27)[name = tensor("op_1003")]; + tensor filled_window_27 = add(x = var_1002, y = var_1003)[name = tensor("filled_window_27")]; + tensor var_1005 = greater_equal(x = clip_25, y = var_17)[name = tensor("op_1005")]; + tensor var_1006_dtype_0 = const()[name = tensor("op_1006_dtype_0"), val = tensor("fp32")]; + tensor var_1007 = const()[name = tensor("op_1007"), val = tensor([1, 1, 1])]; + tensor var_1006 = cast(dtype = var_1006_dtype_0, x = var_1005)[name = tensor("cast_164")]; + tensor use_roll_27 = reshape(shape = var_1007, x = var_1006)[name = tensor("use_roll_27")]; + tensor var_1009 = sub(x = var_18, y = use_roll_27)[name = tensor("op_1009")]; + tensor var_1010 = mul(x = filled_window_27, y = var_1009)[name = tensor("op_1010")]; + tensor var_1011 = mul(x = rolled_27, y = use_roll_27)[name = tensor("op_1011")]; + tensor window_33 = add(x = var_1010, y = var_1011)[name = tensor("window_33")]; + tensor var_1013 = const()[name = tensor("op_1013"), val = tensor(0x1p+0)]; + tensor var_1014 = add(x = clip_25, y = var_1013)[name = tensor("op_1014")]; + tensor clip_27 = clip(alpha = var_22, beta = var_17, x = var_1014)[name = tensor("clip_27")]; + tensor var_1017_begin_0 = const()[name = tensor("op_1017_begin_0"), val = tensor([0, 4, 0])]; + tensor var_1017_end_0 = const()[name = tensor("op_1017_end_0"), val = tensor([1, 1, 256])]; + tensor var_1017_end_mask_0 = const()[name = tensor("op_1017_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1017 = slice_by_index(begin = var_1017_begin_0, end = var_1017_end_0, end_mask = var_1017_end_mask_0, x = x_15)[name = tensor("op_1017")]; + tensor var_1020_begin_0 = const()[name = tensor("op_1020_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1020_end_0 = const()[name = tensor("op_1020_end_0"), val = tensor([1, 16, 256])]; + tensor var_1020_end_mask_0 = const()[name = tensor("op_1020_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1020 = slice_by_index(begin = var_1020_begin_0, end = var_1020_end_0, end_mask = var_1020_end_mask_0, x = window_33)[name = tensor("op_1020")]; tensor rolled_29_interleave_0 = const()[name = tensor("rolled_29_interleave_0"), val = tensor(false)]; - tensor rolled_29 = concat(axis = var_34, interleave = rolled_29_interleave_0, values = (var_1666, var_1663))[name = tensor("rolled_29")]; + tensor rolled_29 = concat(axis = var_34, interleave = rolled_29_interleave_0, values = (var_1020, var_1017))[name = tensor("rolled_29")]; tensor clip_28 = clip(alpha = var_22, beta = var_21, x = clip_27)[name = tensor("clip_28")]; - tensor var_1671 = equal(x = encoder__conv_positions, y = clip_28)[name = tensor("op_1671")]; - tensor var_1672_dtype_0 = const()[name = tensor("op_1672_dtype_0"), val = tensor("fp32")]; - tensor var_1673 = const()[name = tensor("op_1673"), val = tensor([1, 16, 1])]; - tensor var_1672 = cast(dtype = var_1672_dtype_0, x = var_1671)[name = tensor("cast_163")]; - tensor fill_mask_29 = reshape(shape = var_1673, x = var_1672)[name = tensor("fill_mask_29")]; + tensor var_1025 = equal(x = encoder__conv_positions, y = clip_28)[name = tensor("op_1025")]; + tensor var_1026_dtype_0 = const()[name = tensor("op_1026_dtype_0"), val = tensor("fp32")]; + tensor var_1027 = const()[name = tensor("op_1027"), val = tensor([1, 16, 1])]; + tensor var_1026 = cast(dtype = var_1026_dtype_0, x = var_1025)[name = tensor("cast_163")]; + tensor fill_mask_29 = reshape(shape = var_1027, x = var_1026)[name = tensor("fill_mask_29")]; tensor x_expanded_29_reps_0 = const()[name = tensor("x_expanded_29_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_29 = tile(reps = x_expanded_29_reps_0, x = var_1663)[name = tensor("x_expanded_29")]; - tensor var_1677 = sub(x = var_18, y = fill_mask_29)[name = tensor("op_1677")]; - tensor var_1678 = mul(x = input_211, y = var_1677)[name = tensor("op_1678")]; - tensor var_1679 = mul(x = x_expanded_29, y = fill_mask_29)[name = tensor("op_1679")]; - tensor filled_window_29 = add(x = var_1678, y = var_1679)[name = tensor("filled_window_29")]; - tensor var_1681 = greater_equal(x = clip_27, y = var_17)[name = tensor("op_1681")]; - tensor var_1682_dtype_0 = const()[name = tensor("op_1682_dtype_0"), val = tensor("fp32")]; - tensor var_1683 = const()[name = tensor("op_1683"), val = tensor([1, 1, 1])]; - tensor var_1682 = cast(dtype = var_1682_dtype_0, x = var_1681)[name = tensor("cast_162")]; - tensor use_roll_29 = reshape(shape = var_1683, x = var_1682)[name = tensor("use_roll_29")]; - tensor var_1685 = sub(x = var_18, y = use_roll_29)[name = tensor("op_1685")]; - tensor var_1686 = mul(x = filled_window_29, y = var_1685)[name = tensor("op_1686")]; - tensor var_1687 = mul(x = rolled_29, y = use_roll_29)[name = tensor("op_1687")]; - tensor input_221 = add(x = var_1686, y = var_1687)[name = tensor("input_221")]; - tensor var_1689 = const()[name = tensor("op_1689"), val = tensor(0x1p+0)]; - tensor var_1690 = add(x = clip_27, y = var_1689)[name = tensor("op_1690")]; - tensor clip_29 = clip(alpha = var_22, beta = var_17, x = var_1690)[name = tensor("clip_29")]; - tensor x_41_axes_0 = const()[name = tensor("x_41_axes_0"), val = tensor([-1])]; - tensor x_41 = layer_norm(axes = x_41_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_221)[name = tensor("x_41")]; - tensor input_223_perm_0 = const()[name = tensor("input_223_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_95_pad_type_0 = const()[name = tensor("inputs_95_pad_type_0"), val = tensor("valid")]; - tensor inputs_95_strides_0 = const()[name = tensor("inputs_95_strides_0"), val = tensor([1])]; - tensor inputs_95_pad_0 = const()[name = tensor("inputs_95_pad_0"), val = tensor([0, 0])]; - tensor inputs_95_dilations_0 = const()[name = tensor("inputs_95_dilations_0"), val = tensor([1])]; - tensor inputs_95_groups_0 = const()[name = tensor("inputs_95_groups_0"), val = tensor(1)]; - tensor input_223 = transpose(perm = input_223_perm_0, x = x_41)[name = tensor("transpose_46")]; - tensor inputs_95 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_95_dilations_0, groups = inputs_95_groups_0, pad = inputs_95_pad_0, pad_type = inputs_95_pad_type_0, strides = inputs_95_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_223)[name = tensor("inputs_95")]; - tensor var_1711_split_sizes_0 = const()[name = tensor("op_1711_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1711_axis_0 = const()[name = tensor("op_1711_axis_0"), val = tensor(1)]; - tensor var_1711_0, tensor var_1711_1 = split(axis = var_1711_axis_0, split_sizes = var_1711_split_sizes_0, x = inputs_95)[name = tensor("op_1711")]; - tensor var_1713 = sigmoid(x = var_1711_1)[name = tensor("op_1713")]; - tensor inputs_97 = mul(x = var_1711_0, y = var_1713)[name = tensor("inputs_97")]; - tensor outputs_aug_29_pad_type_0 = const()[name = tensor("outputs_aug_29_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_29_pad_0 = const()[name = tensor("outputs_aug_29_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_29_groups_0 = const()[name = tensor("outputs_aug_29_groups_0"), val = tensor(256)]; - tensor outputs_aug_29_strides_0 = const()[name = tensor("outputs_aug_29_strides_0"), val = tensor([1])]; - tensor outputs_aug_29_dilations_0 = const()[name = tensor("outputs_aug_29_dilations_0"), val = tensor([1])]; - tensor outputs_aug_29 = conv(dilations = outputs_aug_29_dilations_0, groups = outputs_aug_29_groups_0, pad = outputs_aug_29_pad_0, pad_type = outputs_aug_29_pad_type_0, strides = outputs_aug_29_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_97)[name = tensor("outputs_aug_29")]; - tensor input_225_begin_0 = const()[name = tensor("input_225_begin_0"), val = tensor([0, 0, 0])]; - tensor input_225_end_0 = const()[name = tensor("input_225_end_0"), val = tensor([1, 256, 16])]; - tensor input_225_end_mask_0 = const()[name = tensor("input_225_end_mask_0"), val = tensor([true, true, false])]; - tensor input_225 = slice_by_index(begin = input_225_begin_0, end = input_225_end_0, end_mask = input_225_end_mask_0, x = outputs_aug_29)[name = tensor("input_225")]; - tensor inputs_99 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_225)[name = tensor("inputs_99")]; - tensor input_227 = silu(x = inputs_99)[name = tensor("input_227")]; - tensor input_229_pad_type_0 = const()[name = tensor("input_229_pad_type_0"), val = tensor("valid")]; - tensor input_229_strides_0 = const()[name = tensor("input_229_strides_0"), val = tensor([1])]; - tensor input_229_pad_0 = const()[name = tensor("input_229_pad_0"), val = tensor([0, 0])]; - tensor input_229_dilations_0 = const()[name = tensor("input_229_dilations_0"), val = tensor([1])]; - tensor input_229_groups_0 = const()[name = tensor("input_229_groups_0"), val = tensor(1)]; - tensor input_229 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_229_dilations_0, groups = input_229_groups_0, pad = input_229_pad_0, pad_type = input_229_pad_type_0, strides = input_229_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_227)[name = tensor("input_229")]; - tensor conv_out_29_perm_0 = const()[name = tensor("conv_out_29_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1744_begin_0 = const()[name = tensor("op_1744_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1744_end_0 = const()[name = tensor("op_1744_end_0"), val = tensor([1, 16, 256])]; - tensor var_1744_end_mask_0 = const()[name = tensor("op_1744_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_29 = transpose(perm = conv_out_29_perm_0, x = input_229)[name = tensor("transpose_45")]; - tensor var_1744 = slice_by_index(begin = var_1744_begin_0, end = var_1744_end_0, end_mask = var_1744_end_mask_0, x = conv_out_29)[name = tensor("op_1744")]; - tensor var_1747_interleave_0 = const()[name = tensor("op_1747_interleave_0"), val = tensor(false)]; - tensor var_1747 = concat(axis = var_34, interleave = var_1747_interleave_0, values = (var_1408, var_1492, var_1576, var_1660, var_1744))[name = tensor("op_1747")]; - tensor input_231 = add(x = x_31, y = var_1747)[name = tensor("input_231")]; - tensor input_233_axes_0 = const()[name = tensor("input_233_axes_0"), val = tensor([-1])]; - tensor input_233 = layer_norm(axes = input_233_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_231)[name = tensor("input_233")]; - tensor inputs_101 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_233)[name = tensor("linear_26")]; - tensor input_235 = silu(x = inputs_101)[name = tensor("input_235")]; - tensor input_239 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_235)[name = tensor("linear_27")]; - tensor var_1770 = const()[name = tensor("op_1770"), val = tensor(0x1p-1)]; - tensor var_1771 = mul(x = input_239, y = var_1770)[name = tensor("op_1771")]; - tensor input_241 = add(x = var_1771, y = input_231)[name = tensor("input_241")]; - tensor input_243_axes_0 = const()[name = tensor("input_243_axes_0"), val = tensor([-1])]; - tensor input_243 = layer_norm(axes = input_243_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_36, gamma = encoder_layer_norm_2_weight, x = input_241)[name = tensor("input_243")]; - tensor input_245_axes_0 = const()[name = tensor("input_245_axes_0"), val = tensor([-1])]; - tensor input_245 = layer_norm(axes = input_245_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_243)[name = tensor("input_245")]; - tensor inputs_103 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_245)[name = tensor("linear_28")]; - tensor input_247 = silu(x = inputs_103)[name = tensor("input_247")]; - tensor input_251 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_247)[name = tensor("linear_29")]; - tensor var_1800 = const()[name = tensor("op_1800"), val = tensor(0x1p-1)]; - tensor var_1801 = mul(x = input_251, y = var_1800)[name = tensor("op_1801")]; - tensor input_253 = add(x = var_1801, y = input_243)[name = tensor("input_253")]; - tensor x_43_axes_0 = const()[name = tensor("x_43_axes_0"), val = tensor([-1])]; - tensor x_43 = layer_norm(axes = x_43_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_36, gamma = encoder_ret_lns_3_weight, x = input_253)[name = tensor("x_43")]; + tensor x_expanded_29 = tile(reps = x_expanded_29_reps_0, x = var_1017)[name = tensor("x_expanded_29")]; + tensor var_1031 = sub(x = var_18, y = fill_mask_29)[name = tensor("op_1031")]; + tensor var_1032 = mul(x = window_33, y = var_1031)[name = tensor("op_1032")]; + tensor var_1033 = mul(x = x_expanded_29, y = fill_mask_29)[name = tensor("op_1033")]; + tensor filled_window_29 = add(x = var_1032, y = var_1033)[name = tensor("filled_window_29")]; + tensor var_1035 = greater_equal(x = clip_27, y = var_17)[name = tensor("op_1035")]; + tensor var_1036_dtype_0 = const()[name = tensor("op_1036_dtype_0"), val = tensor("fp32")]; + tensor var_1037 = const()[name = tensor("op_1037"), val = tensor([1, 1, 1])]; + tensor var_1036 = cast(dtype = var_1036_dtype_0, x = var_1035)[name = tensor("cast_162")]; + tensor use_roll_29 = reshape(shape = var_1037, x = var_1036)[name = tensor("use_roll_29")]; + tensor var_1039 = sub(x = var_18, y = use_roll_29)[name = tensor("op_1039")]; + tensor var_1040 = mul(x = filled_window_29, y = var_1039)[name = tensor("op_1040")]; + tensor var_1041 = mul(x = rolled_29, y = use_roll_29)[name = tensor("op_1041")]; + tensor window_35 = add(x = var_1040, y = var_1041)[name = tensor("window_35")]; + tensor var_1043 = const()[name = tensor("op_1043"), val = tensor(0x1p+0)]; + tensor var_1044 = add(x = clip_27, y = var_1043)[name = tensor("op_1044")]; + tensor clip_29 = clip(alpha = var_22, beta = var_17, x = var_1044)[name = tensor("clip_29")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_31, interleave = input_101_interleave_0, values = (window_27, window_29, window_31, window_33, window_35))[name = tensor("input_101")]; + tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; + tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; + tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_23_pad_type_0 = const()[name = tensor("inputs_23_pad_type_0"), val = tensor("valid")]; + tensor inputs_23_strides_0 = const()[name = tensor("inputs_23_strides_0"), val = tensor([1])]; + tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; + tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; + tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; + tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; + tensor var_1067_split_sizes_0 = const()[name = tensor("op_1067_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1067_axis_0 = const()[name = tensor("op_1067_axis_0"), val = tensor(1)]; + tensor var_1067_0, tensor var_1067_1 = split(axis = var_1067_axis_0, split_sizes = var_1067_split_sizes_0, x = inputs_23)[name = tensor("op_1067")]; + tensor var_1069 = sigmoid(x = var_1067_1)[name = tensor("op_1069")]; + tensor inputs_25 = mul(x = var_1067_0, y = var_1069)[name = tensor("inputs_25")]; + tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; + tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; + tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; + tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_25)[name = tensor("outputs_aug_5")]; + tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; + tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([5, 256, 16])]; + tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; + tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_5)[name = tensor("input_105")]; + tensor inputs_27 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_105)[name = tensor("inputs_27")]; + tensor input_107 = silu(x = inputs_27)[name = tensor("input_107")]; + tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; + tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; + tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; + tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; + tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; + tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; + tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1100_begin_0 = const()[name = tensor("op_1100_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1100_end_0 = const()[name = tensor("op_1100_end_0"), val = tensor([5, 16, 256])]; + tensor var_1100_end_mask_0 = const()[name = tensor("op_1100_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_1100 = slice_by_index(begin = var_1100_begin_0, end = var_1100_end_0, end_mask = var_1100_end_mask_0, x = conv_out_5)[name = tensor("op_1100")]; + tensor var_1102_perm_0 = const()[name = tensor("op_1102_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1102 = transpose(perm = var_1102_perm_0, x = var_1100)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_1102)[name = tensor("input_111")]; + tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; + tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; + tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; + tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; + tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; + tensor var_1125 = const()[name = tensor("op_1125"), val = tensor(0x1p-1)]; + tensor var_1126 = mul(x = input_119, y = var_1125)[name = tensor("op_1126")]; + tensor input_121 = add(x = var_1126, y = input_111)[name = tensor("input_121")]; + tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; + tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_36, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; + tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; + tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; + tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; + tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; + tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; + tensor var_1155 = const()[name = tensor("op_1155"), val = tensor(0x1p-1)]; + tensor var_1156 = mul(x = input_131, y = var_1155)[name = tensor("op_1156")]; + tensor input_133 = add(x = var_1156, y = input_123)[name = tensor("input_133")]; + tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; + tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_36, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; tensor prev_kv_7_end_0 = const()[name = tensor("prev_kv_7_end_0"), val = tensor([4, 1, 4, 64, 64])]; tensor prev_kv_7_end_mask_0 = const()[name = tensor("prev_kv_7_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1641,455 +1180,301 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_1815 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_43)[name = tensor("linear_30")]; - tensor var_1816 = const()[name = tensor("op_1816"), val = tensor([1, 5, 4, 64])]; - tensor var_1817 = reshape(shape = var_1816, x = var_1815)[name = tensor("op_1817")]; + tensor var_1170 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_1171 = const()[name = tensor("op_1171"), val = tensor([1, 5, 4, 64])]; + tensor var_1172 = reshape(shape = var_1171, x = var_1170)[name = tensor("op_1172")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1821 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_43)[name = tensor("linear_31")]; - tensor var_1822 = const()[name = tensor("op_1822"), val = tensor(0x1p-3)]; - tensor var_1823 = mul(x = var_1821, y = var_1822)[name = tensor("op_1823")]; - tensor var_1824 = const()[name = tensor("op_1824"), val = tensor([1, 5, 4, 64])]; - tensor var_1825 = reshape(shape = var_1824, x = var_1823)[name = tensor("op_1825")]; + tensor var_1176 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_1177 = const()[name = tensor("op_1177"), val = tensor(0x1p-3)]; + tensor var_1178 = mul(x = var_1176, y = var_1177)[name = tensor("op_1178")]; + tensor var_1179 = const()[name = tensor("op_1179"), val = tensor([1, 5, 4, 64])]; + tensor var_1180 = reshape(shape = var_1179, x = var_1178)[name = tensor("op_1180")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1829 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_43)[name = tensor("linear_32")]; - tensor var_1830 = const()[name = tensor("op_1830"), val = tensor([1, 5, 4, 64])]; - tensor var_1831 = reshape(shape = var_1830, x = var_1829)[name = tensor("op_1831")]; + tensor var_1184 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_1185 = const()[name = tensor("op_1185"), val = tensor([1, 5, 4, 64])]; + tensor var_1186 = reshape(shape = var_1185, x = var_1184)[name = tensor("op_1186")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_257 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_43)[name = tensor("linear_33")]; + tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; tensor s_t_7 = add(x = prev_scale_7, y = encoder__t_index)[name = tensor("s_t_7")]; tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_1825)[name = tensor("transpose_43")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_1817)[name = tensor("transpose_44")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_1180)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_1172)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_1841 = const()[name = tensor("op_1841"), val = tensor([5, 1])]; - tensor var_1842 = reshape(shape = var_1841, x = sqrt_s_t_7)[name = tensor("op_1842")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_1842)[name = tensor("M_7")]; - tensor var_1844 = mul(x = qk_7, y = M_7)[name = tensor("op_1844")]; + tensor var_1196 = const()[name = tensor("op_1196"), val = tensor([5, 1])]; + tensor var_1197 = reshape(shape = var_1196, x = sqrt_s_t_7)[name = tensor("op_1197")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_1197)[name = tensor("M_7")]; + tensor var_1199 = mul(x = qk_7, y = M_7)[name = tensor("op_1199")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_1831)[name = tensor("transpose_42")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1844, y = v_7)[name = tensor("inner_7")]; - tensor var_1846_transpose_x_0 = const()[name = tensor("op_1846_transpose_x_0"), val = tensor(false)]; - tensor var_1846_transpose_y_0 = const()[name = tensor("op_1846_transpose_y_0"), val = tensor(false)]; - tensor var_1846 = matmul(transpose_x = var_1846_transpose_x_0, transpose_y = var_1846_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1846")]; - tensor var_1847 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1847")]; - tensor var_1848 = const()[name = tensor("op_1848"), val = tensor([1, 1, 5, 1])]; - tensor var_1849 = reshape(shape = var_1848, x = var_1847)[name = tensor("op_1849")]; - tensor cross_7 = mul(x = var_1846, y = var_1849)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_1186)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1199, y = v_7)[name = tensor("inner_7")]; + tensor var_1201_transpose_x_0 = const()[name = tensor("op_1201_transpose_x_0"), val = tensor(false)]; + tensor var_1201_transpose_y_0 = const()[name = tensor("op_1201_transpose_y_0"), val = tensor(false)]; + tensor var_1201 = matmul(transpose_x = var_1201_transpose_x_0, transpose_y = var_1201_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1201")]; + tensor var_1202 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1202")]; + tensor var_1203 = const()[name = tensor("op_1203"), val = tensor([1, 1, 5, 1])]; + tensor var_1204 = reshape(shape = var_1203, x = var_1202)[name = tensor("op_1204")]; + tensor cross_7 = mul(x = var_1201, y = var_1204)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_1852 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1852")]; - tensor var_1854_transpose_x_1 = const()[name = tensor("op_1854_transpose_x_1"), val = tensor(true)]; - tensor var_1854_transpose_y_1 = const()[name = tensor("op_1854_transpose_y_1"), val = tensor(false)]; - tensor var_1854 = matmul(transpose_x = var_1854_transpose_x_1, transpose_y = var_1854_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1854")]; - tensor new_kv_unnorm_7 = add(x = var_1852, y = var_1854)[name = tensor("new_kv_unnorm_7")]; - tensor var_1856 = const()[name = tensor("op_1856"), val = tensor(0x1.4p+2)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_1856)[name = tensor("new_scale_7")]; - tensor var_1858 = sqrt(x = new_scale_7)[name = tensor("op_1858")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1858)[name = tensor("nkv_1")]; - tensor var_1860_perm_0 = const()[name = tensor("op_1860_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1207 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1207")]; + tensor var_1209_transpose_x_1 = const()[name = tensor("op_1209_transpose_x_1"), val = tensor(true)]; + tensor var_1209_transpose_y_1 = const()[name = tensor("op_1209_transpose_y_1"), val = tensor(false)]; + tensor var_1209 = matmul(transpose_x = var_1209_transpose_x_1, transpose_y = var_1209_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1209")]; + tensor new_kv_unnorm_7 = add(x = var_1207, y = var_1209)[name = tensor("new_kv_unnorm_7")]; + tensor var_1211 = const()[name = tensor("op_1211"), val = tensor(0x1.4p+2)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_1211)[name = tensor("new_scale_7")]; + tensor var_1213 = sqrt(x = new_scale_7)[name = tensor("op_1213")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1213)[name = tensor("nkv_1")]; + tensor var_1215_perm_0 = const()[name = tensor("op_1215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_1860 = transpose(perm = var_1860_perm_0, x = out_19)[name = tensor("transpose_41")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_25, x = var_1860)[name = tensor("out_21")]; - tensor var_1864 = const()[name = tensor("op_1864"), val = tensor([1, 5, 256])]; - tensor out_23 = reshape(shape = var_1864, x = out_21)[name = tensor("out_23")]; - tensor var_1866 = silu(x = input_257)[name = tensor("op_1866")]; - tensor input_259 = mul(x = var_1866, y = out_23)[name = tensor("input_259")]; - tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_259)[name = tensor("linear_34")]; - tensor x_45 = add(x = input_253, y = ret_out_7)[name = tensor("x_45")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor var_1215 = transpose(perm = var_1215_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_25, x = var_1215)[name = tensor("out_21")]; + tensor var_1219 = const()[name = tensor("op_1219"), val = tensor([1, 5, 256])]; + tensor out_23 = reshape(shape = var_1219, x = out_21)[name = tensor("out_23")]; + tensor var_1221 = silu(x = input_137)[name = tensor("op_1221")]; + tensor input_139 = mul(x = var_1221, y = out_23)[name = tensor("input_139")]; + tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; + tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; + tensor window_37_begin_0 = const()[name = tensor("window_37_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_37_end_0 = const()[name = tensor("window_37_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_37_end_mask_0 = const()[name = tensor("window_37_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_37_squeeze_mask_0 = const()[name = tensor("window_37_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_37 = slice_by_index(begin = window_37_begin_0, end = window_37_end_0, end_mask = window_37_end_mask_0, squeeze_mask = window_37_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_37")]; tensor filled_31_begin_0 = const()[name = tensor("filled_31_begin_0"), val = tensor([3, 0])]; tensor filled_31_end_0 = const()[name = tensor("filled_31_end_0"), val = tensor([4, 1])]; tensor filled_31_end_mask_0 = const()[name = tensor("filled_31_end_mask_0"), val = tensor([false, true])]; tensor filled_31_squeeze_mask_0 = const()[name = tensor("filled_31_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_31 = slice_by_index(begin = filled_31_begin_0, end = filled_31_end_0, end_mask = filled_31_end_mask_0, squeeze_mask = filled_31_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_31")]; - tensor var_1875_begin_0 = const()[name = tensor("op_1875_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1875_end_0 = const()[name = tensor("op_1875_end_0"), val = tensor([1, 1, 256])]; - tensor var_1875_end_mask_0 = const()[name = tensor("op_1875_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1875 = slice_by_index(begin = var_1875_begin_0, end = var_1875_end_0, end_mask = var_1875_end_mask_0, x = x_45)[name = tensor("op_1875")]; - tensor var_1878_begin_0 = const()[name = tensor("op_1878_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1878_end_0 = const()[name = tensor("op_1878_end_0"), val = tensor([1, 16, 256])]; - tensor var_1878_end_mask_0 = const()[name = tensor("op_1878_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1878 = slice_by_index(begin = var_1878_begin_0, end = var_1878_end_0, end_mask = var_1878_end_mask_0, x = window)[name = tensor("op_1878")]; + tensor var_1230_begin_0 = const()[name = tensor("op_1230_begin_0"), val = tensor([0, 0, 0])]; + tensor var_1230_end_0 = const()[name = tensor("op_1230_end_0"), val = tensor([1, 1, 256])]; + tensor var_1230_end_mask_0 = const()[name = tensor("op_1230_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1230 = slice_by_index(begin = var_1230_begin_0, end = var_1230_end_0, end_mask = var_1230_end_mask_0, x = x_21)[name = tensor("op_1230")]; + tensor var_1233_begin_0 = const()[name = tensor("op_1233_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1233_end_0 = const()[name = tensor("op_1233_end_0"), val = tensor([1, 16, 256])]; + tensor var_1233_end_mask_0 = const()[name = tensor("op_1233_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1233 = slice_by_index(begin = var_1233_begin_0, end = var_1233_end_0, end_mask = var_1233_end_mask_0, x = window_37)[name = tensor("op_1233")]; tensor rolled_31_interleave_0 = const()[name = tensor("rolled_31_interleave_0"), val = tensor(false)]; - tensor rolled_31 = concat(axis = var_34, interleave = rolled_31_interleave_0, values = (var_1878, var_1875))[name = tensor("rolled_31")]; + tensor rolled_31 = concat(axis = var_34, interleave = rolled_31_interleave_0, values = (var_1233, var_1230))[name = tensor("rolled_31")]; tensor clip_30 = clip(alpha = var_22, beta = var_21, x = filled_31)[name = tensor("clip_30")]; - tensor var_1883 = equal(x = encoder__conv_positions, y = clip_30)[name = tensor("op_1883")]; - tensor var_1884_dtype_0 = const()[name = tensor("op_1884_dtype_0"), val = tensor("fp32")]; - tensor var_1885 = const()[name = tensor("op_1885"), val = tensor([1, 16, 1])]; - tensor var_1884 = cast(dtype = var_1884_dtype_0, x = var_1883)[name = tensor("cast_161")]; - tensor fill_mask_31 = reshape(shape = var_1885, x = var_1884)[name = tensor("fill_mask_31")]; + tensor var_1238 = equal(x = encoder__conv_positions, y = clip_30)[name = tensor("op_1238")]; + tensor var_1239_dtype_0 = const()[name = tensor("op_1239_dtype_0"), val = tensor("fp32")]; + tensor var_1240 = const()[name = tensor("op_1240"), val = tensor([1, 16, 1])]; + tensor var_1239 = cast(dtype = var_1239_dtype_0, x = var_1238)[name = tensor("cast_161")]; + tensor fill_mask_31 = reshape(shape = var_1240, x = var_1239)[name = tensor("fill_mask_31")]; tensor x_expanded_31_reps_0 = const()[name = tensor("x_expanded_31_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_31 = tile(reps = x_expanded_31_reps_0, x = var_1875)[name = tensor("x_expanded_31")]; - tensor var_1889 = sub(x = var_18, y = fill_mask_31)[name = tensor("op_1889")]; - tensor var_1890 = mul(x = window, y = var_1889)[name = tensor("op_1890")]; - tensor var_1891 = mul(x = x_expanded_31, y = fill_mask_31)[name = tensor("op_1891")]; - tensor filled_window_31 = add(x = var_1890, y = var_1891)[name = tensor("filled_window_31")]; - tensor var_1893 = greater_equal(x = filled_31, y = var_17)[name = tensor("op_1893")]; - tensor var_1894_dtype_0 = const()[name = tensor("op_1894_dtype_0"), val = tensor("fp32")]; - tensor var_1895 = const()[name = tensor("op_1895"), val = tensor([1, 1, 1])]; - tensor var_1894 = cast(dtype = var_1894_dtype_0, x = var_1893)[name = tensor("cast_160")]; - tensor use_roll_31 = reshape(shape = var_1895, x = var_1894)[name = tensor("use_roll_31")]; - tensor var_1897 = sub(x = var_18, y = use_roll_31)[name = tensor("op_1897")]; - tensor var_1898 = mul(x = filled_window_31, y = var_1897)[name = tensor("op_1898")]; - tensor var_1899 = mul(x = rolled_31, y = use_roll_31)[name = tensor("op_1899")]; - tensor input_261 = add(x = var_1898, y = var_1899)[name = tensor("input_261")]; - tensor var_1901 = const()[name = tensor("op_1901"), val = tensor(0x1p+0)]; - tensor var_1902 = add(x = filled_31, y = var_1901)[name = tensor("op_1902")]; - tensor clip_31 = clip(alpha = var_22, beta = var_17, x = var_1902)[name = tensor("clip_31")]; - tensor x_47_axes_0 = const()[name = tensor("x_47_axes_0"), val = tensor([-1])]; - tensor x_47 = layer_norm(axes = x_47_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_261)[name = tensor("x_47")]; - tensor input_263_perm_0 = const()[name = tensor("input_263_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_105_pad_type_0 = const()[name = tensor("inputs_105_pad_type_0"), val = tensor("valid")]; - tensor inputs_105_strides_0 = const()[name = tensor("inputs_105_strides_0"), val = tensor([1])]; - tensor inputs_105_pad_0 = const()[name = tensor("inputs_105_pad_0"), val = tensor([0, 0])]; - tensor inputs_105_dilations_0 = const()[name = tensor("inputs_105_dilations_0"), val = tensor([1])]; - tensor inputs_105_groups_0 = const()[name = tensor("inputs_105_groups_0"), val = tensor(1)]; - tensor input_263 = transpose(perm = input_263_perm_0, x = x_47)[name = tensor("transpose_40")]; - tensor inputs_105 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_105_dilations_0, groups = inputs_105_groups_0, pad = inputs_105_pad_0, pad_type = inputs_105_pad_type_0, strides = inputs_105_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_263)[name = tensor("inputs_105")]; - tensor var_1923_split_sizes_0 = const()[name = tensor("op_1923_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1923_axis_0 = const()[name = tensor("op_1923_axis_0"), val = tensor(1)]; - tensor var_1923_0, tensor var_1923_1 = split(axis = var_1923_axis_0, split_sizes = var_1923_split_sizes_0, x = inputs_105)[name = tensor("op_1923")]; - tensor var_1925 = sigmoid(x = var_1923_1)[name = tensor("op_1925")]; - tensor inputs_107 = mul(x = var_1923_0, y = var_1925)[name = tensor("inputs_107")]; - tensor outputs_aug_31_pad_type_0 = const()[name = tensor("outputs_aug_31_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_31_pad_0 = const()[name = tensor("outputs_aug_31_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_31_groups_0 = const()[name = tensor("outputs_aug_31_groups_0"), val = tensor(256)]; - tensor outputs_aug_31_strides_0 = const()[name = tensor("outputs_aug_31_strides_0"), val = tensor([1])]; - tensor outputs_aug_31_dilations_0 = const()[name = tensor("outputs_aug_31_dilations_0"), val = tensor([1])]; - tensor outputs_aug_31 = conv(dilations = outputs_aug_31_dilations_0, groups = outputs_aug_31_groups_0, pad = outputs_aug_31_pad_0, pad_type = outputs_aug_31_pad_type_0, strides = outputs_aug_31_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_107)[name = tensor("outputs_aug_31")]; - tensor input_265_begin_0 = const()[name = tensor("input_265_begin_0"), val = tensor([0, 0, 0])]; - tensor input_265_end_0 = const()[name = tensor("input_265_end_0"), val = tensor([1, 256, 16])]; - tensor input_265_end_mask_0 = const()[name = tensor("input_265_end_mask_0"), val = tensor([true, true, false])]; - tensor input_265 = slice_by_index(begin = input_265_begin_0, end = input_265_end_0, end_mask = input_265_end_mask_0, x = outputs_aug_31)[name = tensor("input_265")]; - tensor inputs_109 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_265)[name = tensor("inputs_109")]; - tensor input_267 = silu(x = inputs_109)[name = tensor("input_267")]; - tensor input_269_pad_type_0 = const()[name = tensor("input_269_pad_type_0"), val = tensor("valid")]; - tensor input_269_strides_0 = const()[name = tensor("input_269_strides_0"), val = tensor([1])]; - tensor input_269_pad_0 = const()[name = tensor("input_269_pad_0"), val = tensor([0, 0])]; - tensor input_269_dilations_0 = const()[name = tensor("input_269_dilations_0"), val = tensor([1])]; - tensor input_269_groups_0 = const()[name = tensor("input_269_groups_0"), val = tensor(1)]; - tensor input_269 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_269_dilations_0, groups = input_269_groups_0, pad = input_269_pad_0, pad_type = input_269_pad_type_0, strides = input_269_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_267)[name = tensor("input_269")]; - tensor conv_out_31_perm_0 = const()[name = tensor("conv_out_31_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1956_begin_0 = const()[name = tensor("op_1956_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1956_end_0 = const()[name = tensor("op_1956_end_0"), val = tensor([1, 16, 256])]; - tensor var_1956_end_mask_0 = const()[name = tensor("op_1956_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_31 = transpose(perm = conv_out_31_perm_0, x = input_269)[name = tensor("transpose_39")]; - tensor var_1956 = slice_by_index(begin = var_1956_begin_0, end = var_1956_end_0, end_mask = var_1956_end_mask_0, x = conv_out_31)[name = tensor("op_1956")]; - tensor var_1959_begin_0 = const()[name = tensor("op_1959_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1959_end_0 = const()[name = tensor("op_1959_end_0"), val = tensor([1, 2, 256])]; - tensor var_1959_end_mask_0 = const()[name = tensor("op_1959_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1959 = slice_by_index(begin = var_1959_begin_0, end = var_1959_end_0, end_mask = var_1959_end_mask_0, x = x_45)[name = tensor("op_1959")]; - tensor var_1962_begin_0 = const()[name = tensor("op_1962_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1962_end_0 = const()[name = tensor("op_1962_end_0"), val = tensor([1, 16, 256])]; - tensor var_1962_end_mask_0 = const()[name = tensor("op_1962_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1962 = slice_by_index(begin = var_1962_begin_0, end = var_1962_end_0, end_mask = var_1962_end_mask_0, x = input_261)[name = tensor("op_1962")]; + tensor x_expanded_31 = tile(reps = x_expanded_31_reps_0, x = var_1230)[name = tensor("x_expanded_31")]; + tensor var_1244 = sub(x = var_18, y = fill_mask_31)[name = tensor("op_1244")]; + tensor var_1245 = mul(x = window_37, y = var_1244)[name = tensor("op_1245")]; + tensor var_1246 = mul(x = x_expanded_31, y = fill_mask_31)[name = tensor("op_1246")]; + tensor filled_window_31 = add(x = var_1245, y = var_1246)[name = tensor("filled_window_31")]; + tensor var_1248 = greater_equal(x = filled_31, y = var_17)[name = tensor("op_1248")]; + tensor var_1249_dtype_0 = const()[name = tensor("op_1249_dtype_0"), val = tensor("fp32")]; + tensor var_1250 = const()[name = tensor("op_1250"), val = tensor([1, 1, 1])]; + tensor var_1249 = cast(dtype = var_1249_dtype_0, x = var_1248)[name = tensor("cast_160")]; + tensor use_roll_31 = reshape(shape = var_1250, x = var_1249)[name = tensor("use_roll_31")]; + tensor var_1252 = sub(x = var_18, y = use_roll_31)[name = tensor("op_1252")]; + tensor var_1253 = mul(x = filled_window_31, y = var_1252)[name = tensor("op_1253")]; + tensor var_1254 = mul(x = rolled_31, y = use_roll_31)[name = tensor("op_1254")]; + tensor window_39 = add(x = var_1253, y = var_1254)[name = tensor("window_39")]; + tensor var_1256 = const()[name = tensor("op_1256"), val = tensor(0x1p+0)]; + tensor var_1257 = add(x = filled_31, y = var_1256)[name = tensor("op_1257")]; + tensor clip_31 = clip(alpha = var_22, beta = var_17, x = var_1257)[name = tensor("clip_31")]; + tensor var_1260_begin_0 = const()[name = tensor("op_1260_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1260_end_0 = const()[name = tensor("op_1260_end_0"), val = tensor([1, 2, 256])]; + tensor var_1260_end_mask_0 = const()[name = tensor("op_1260_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1260 = slice_by_index(begin = var_1260_begin_0, end = var_1260_end_0, end_mask = var_1260_end_mask_0, x = x_21)[name = tensor("op_1260")]; + tensor var_1263_begin_0 = const()[name = tensor("op_1263_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1263_end_0 = const()[name = tensor("op_1263_end_0"), val = tensor([1, 16, 256])]; + tensor var_1263_end_mask_0 = const()[name = tensor("op_1263_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1263 = slice_by_index(begin = var_1263_begin_0, end = var_1263_end_0, end_mask = var_1263_end_mask_0, x = window_39)[name = tensor("op_1263")]; tensor rolled_33_interleave_0 = const()[name = tensor("rolled_33_interleave_0"), val = tensor(false)]; - tensor rolled_33 = concat(axis = var_34, interleave = rolled_33_interleave_0, values = (var_1962, var_1959))[name = tensor("rolled_33")]; + tensor rolled_33 = concat(axis = var_34, interleave = rolled_33_interleave_0, values = (var_1263, var_1260))[name = tensor("rolled_33")]; tensor clip_32 = clip(alpha = var_22, beta = var_21, x = clip_31)[name = tensor("clip_32")]; - tensor var_1967 = equal(x = encoder__conv_positions, y = clip_32)[name = tensor("op_1967")]; - tensor var_1968_dtype_0 = const()[name = tensor("op_1968_dtype_0"), val = tensor("fp32")]; - tensor var_1969 = const()[name = tensor("op_1969"), val = tensor([1, 16, 1])]; - tensor var_1968 = cast(dtype = var_1968_dtype_0, x = var_1967)[name = tensor("cast_159")]; - tensor fill_mask_33 = reshape(shape = var_1969, x = var_1968)[name = tensor("fill_mask_33")]; + tensor var_1268 = equal(x = encoder__conv_positions, y = clip_32)[name = tensor("op_1268")]; + tensor var_1269_dtype_0 = const()[name = tensor("op_1269_dtype_0"), val = tensor("fp32")]; + tensor var_1270 = const()[name = tensor("op_1270"), val = tensor([1, 16, 1])]; + tensor var_1269 = cast(dtype = var_1269_dtype_0, x = var_1268)[name = tensor("cast_159")]; + tensor fill_mask_33 = reshape(shape = var_1270, x = var_1269)[name = tensor("fill_mask_33")]; tensor x_expanded_33_reps_0 = const()[name = tensor("x_expanded_33_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_33 = tile(reps = x_expanded_33_reps_0, x = var_1959)[name = tensor("x_expanded_33")]; - tensor var_1973 = sub(x = var_18, y = fill_mask_33)[name = tensor("op_1973")]; - tensor var_1974 = mul(x = input_261, y = var_1973)[name = tensor("op_1974")]; - tensor var_1975 = mul(x = x_expanded_33, y = fill_mask_33)[name = tensor("op_1975")]; - tensor filled_window_33 = add(x = var_1974, y = var_1975)[name = tensor("filled_window_33")]; - tensor var_1977 = greater_equal(x = clip_31, y = var_17)[name = tensor("op_1977")]; - tensor var_1978_dtype_0 = const()[name = tensor("op_1978_dtype_0"), val = tensor("fp32")]; - tensor var_1979 = const()[name = tensor("op_1979"), val = tensor([1, 1, 1])]; - tensor var_1978 = cast(dtype = var_1978_dtype_0, x = var_1977)[name = tensor("cast_158")]; - tensor use_roll_33 = reshape(shape = var_1979, x = var_1978)[name = tensor("use_roll_33")]; - tensor var_1981 = sub(x = var_18, y = use_roll_33)[name = tensor("op_1981")]; - tensor var_1982 = mul(x = filled_window_33, y = var_1981)[name = tensor("op_1982")]; - tensor var_1983 = mul(x = rolled_33, y = use_roll_33)[name = tensor("op_1983")]; - tensor input_271 = add(x = var_1982, y = var_1983)[name = tensor("input_271")]; - tensor var_1985 = const()[name = tensor("op_1985"), val = tensor(0x1p+0)]; - tensor var_1986 = add(x = clip_31, y = var_1985)[name = tensor("op_1986")]; - tensor clip_33 = clip(alpha = var_22, beta = var_17, x = var_1986)[name = tensor("clip_33")]; - tensor x_49_axes_0 = const()[name = tensor("x_49_axes_0"), val = tensor([-1])]; - tensor x_49 = layer_norm(axes = x_49_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_271)[name = tensor("x_49")]; - tensor input_273_perm_0 = const()[name = tensor("input_273_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_111_pad_type_0 = const()[name = tensor("inputs_111_pad_type_0"), val = tensor("valid")]; - tensor inputs_111_strides_0 = const()[name = tensor("inputs_111_strides_0"), val = tensor([1])]; - tensor inputs_111_pad_0 = const()[name = tensor("inputs_111_pad_0"), val = tensor([0, 0])]; - tensor inputs_111_dilations_0 = const()[name = tensor("inputs_111_dilations_0"), val = tensor([1])]; - tensor inputs_111_groups_0 = const()[name = tensor("inputs_111_groups_0"), val = tensor(1)]; - tensor input_273 = transpose(perm = input_273_perm_0, x = x_49)[name = tensor("transpose_38")]; - tensor inputs_111 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_111_dilations_0, groups = inputs_111_groups_0, pad = inputs_111_pad_0, pad_type = inputs_111_pad_type_0, strides = inputs_111_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_273)[name = tensor("inputs_111")]; - tensor var_2007_split_sizes_0 = const()[name = tensor("op_2007_split_sizes_0"), val = tensor([256, 256])]; - tensor var_2007_axis_0 = const()[name = tensor("op_2007_axis_0"), val = tensor(1)]; - tensor var_2007_0, tensor var_2007_1 = split(axis = var_2007_axis_0, split_sizes = var_2007_split_sizes_0, x = inputs_111)[name = tensor("op_2007")]; - tensor var_2009 = sigmoid(x = var_2007_1)[name = tensor("op_2009")]; - tensor inputs_113 = mul(x = var_2007_0, y = var_2009)[name = tensor("inputs_113")]; - tensor outputs_aug_33_pad_type_0 = const()[name = tensor("outputs_aug_33_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_33_pad_0 = const()[name = tensor("outputs_aug_33_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_33_groups_0 = const()[name = tensor("outputs_aug_33_groups_0"), val = tensor(256)]; - tensor outputs_aug_33_strides_0 = const()[name = tensor("outputs_aug_33_strides_0"), val = tensor([1])]; - tensor outputs_aug_33_dilations_0 = const()[name = tensor("outputs_aug_33_dilations_0"), val = tensor([1])]; - tensor outputs_aug_33 = conv(dilations = outputs_aug_33_dilations_0, groups = outputs_aug_33_groups_0, pad = outputs_aug_33_pad_0, pad_type = outputs_aug_33_pad_type_0, strides = outputs_aug_33_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_113)[name = tensor("outputs_aug_33")]; - tensor input_275_begin_0 = const()[name = tensor("input_275_begin_0"), val = tensor([0, 0, 0])]; - tensor input_275_end_0 = const()[name = tensor("input_275_end_0"), val = tensor([1, 256, 16])]; - tensor input_275_end_mask_0 = const()[name = tensor("input_275_end_mask_0"), val = tensor([true, true, false])]; - tensor input_275 = slice_by_index(begin = input_275_begin_0, end = input_275_end_0, end_mask = input_275_end_mask_0, x = outputs_aug_33)[name = tensor("input_275")]; - tensor inputs_115 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_275)[name = tensor("inputs_115")]; - tensor input_277 = silu(x = inputs_115)[name = tensor("input_277")]; - tensor input_279_pad_type_0 = const()[name = tensor("input_279_pad_type_0"), val = tensor("valid")]; - tensor input_279_strides_0 = const()[name = tensor("input_279_strides_0"), val = tensor([1])]; - tensor input_279_pad_0 = const()[name = tensor("input_279_pad_0"), val = tensor([0, 0])]; - tensor input_279_dilations_0 = const()[name = tensor("input_279_dilations_0"), val = tensor([1])]; - tensor input_279_groups_0 = const()[name = tensor("input_279_groups_0"), val = tensor(1)]; - tensor input_279 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_279_dilations_0, groups = input_279_groups_0, pad = input_279_pad_0, pad_type = input_279_pad_type_0, strides = input_279_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_277)[name = tensor("input_279")]; - tensor conv_out_33_perm_0 = const()[name = tensor("conv_out_33_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2040_begin_0 = const()[name = tensor("op_2040_begin_0"), val = tensor([0, -1, 0])]; - tensor var_2040_end_0 = const()[name = tensor("op_2040_end_0"), val = tensor([1, 16, 256])]; - tensor var_2040_end_mask_0 = const()[name = tensor("op_2040_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_33 = transpose(perm = conv_out_33_perm_0, x = input_279)[name = tensor("transpose_37")]; - tensor var_2040 = slice_by_index(begin = var_2040_begin_0, end = var_2040_end_0, end_mask = var_2040_end_mask_0, x = conv_out_33)[name = tensor("op_2040")]; - tensor var_2043_begin_0 = const()[name = tensor("op_2043_begin_0"), val = tensor([0, 2, 0])]; - tensor var_2043_end_0 = const()[name = tensor("op_2043_end_0"), val = tensor([1, 3, 256])]; - tensor var_2043_end_mask_0 = const()[name = tensor("op_2043_end_mask_0"), val = tensor([true, false, true])]; - tensor var_2043 = slice_by_index(begin = var_2043_begin_0, end = var_2043_end_0, end_mask = var_2043_end_mask_0, x = x_45)[name = tensor("op_2043")]; - tensor var_2046_begin_0 = const()[name = tensor("op_2046_begin_0"), val = tensor([0, 1, 0])]; - tensor var_2046_end_0 = const()[name = tensor("op_2046_end_0"), val = tensor([1, 16, 256])]; - tensor var_2046_end_mask_0 = const()[name = tensor("op_2046_end_mask_0"), val = tensor([true, true, true])]; - tensor var_2046 = slice_by_index(begin = var_2046_begin_0, end = var_2046_end_0, end_mask = var_2046_end_mask_0, x = input_271)[name = tensor("op_2046")]; + tensor x_expanded_33 = tile(reps = x_expanded_33_reps_0, x = var_1260)[name = tensor("x_expanded_33")]; + tensor var_1274 = sub(x = var_18, y = fill_mask_33)[name = tensor("op_1274")]; + tensor var_1275 = mul(x = window_39, y = var_1274)[name = tensor("op_1275")]; + tensor var_1276 = mul(x = x_expanded_33, y = fill_mask_33)[name = tensor("op_1276")]; + tensor filled_window_33 = add(x = var_1275, y = var_1276)[name = tensor("filled_window_33")]; + tensor var_1278 = greater_equal(x = clip_31, y = var_17)[name = tensor("op_1278")]; + tensor var_1279_dtype_0 = const()[name = tensor("op_1279_dtype_0"), val = tensor("fp32")]; + tensor var_1280 = const()[name = tensor("op_1280"), val = tensor([1, 1, 1])]; + tensor var_1279 = cast(dtype = var_1279_dtype_0, x = var_1278)[name = tensor("cast_158")]; + tensor use_roll_33 = reshape(shape = var_1280, x = var_1279)[name = tensor("use_roll_33")]; + tensor var_1282 = sub(x = var_18, y = use_roll_33)[name = tensor("op_1282")]; + tensor var_1283 = mul(x = filled_window_33, y = var_1282)[name = tensor("op_1283")]; + tensor var_1284 = mul(x = rolled_33, y = use_roll_33)[name = tensor("op_1284")]; + tensor window_41 = add(x = var_1283, y = var_1284)[name = tensor("window_41")]; + tensor var_1286 = const()[name = tensor("op_1286"), val = tensor(0x1p+0)]; + tensor var_1287 = add(x = clip_31, y = var_1286)[name = tensor("op_1287")]; + tensor clip_33 = clip(alpha = var_22, beta = var_17, x = var_1287)[name = tensor("clip_33")]; + tensor var_1290_begin_0 = const()[name = tensor("op_1290_begin_0"), val = tensor([0, 2, 0])]; + tensor var_1290_end_0 = const()[name = tensor("op_1290_end_0"), val = tensor([1, 3, 256])]; + tensor var_1290_end_mask_0 = const()[name = tensor("op_1290_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1290 = slice_by_index(begin = var_1290_begin_0, end = var_1290_end_0, end_mask = var_1290_end_mask_0, x = x_21)[name = tensor("op_1290")]; + tensor var_1293_begin_0 = const()[name = tensor("op_1293_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1293_end_0 = const()[name = tensor("op_1293_end_0"), val = tensor([1, 16, 256])]; + tensor var_1293_end_mask_0 = const()[name = tensor("op_1293_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1293 = slice_by_index(begin = var_1293_begin_0, end = var_1293_end_0, end_mask = var_1293_end_mask_0, x = window_41)[name = tensor("op_1293")]; tensor rolled_35_interleave_0 = const()[name = tensor("rolled_35_interleave_0"), val = tensor(false)]; - tensor rolled_35 = concat(axis = var_34, interleave = rolled_35_interleave_0, values = (var_2046, var_2043))[name = tensor("rolled_35")]; + tensor rolled_35 = concat(axis = var_34, interleave = rolled_35_interleave_0, values = (var_1293, var_1290))[name = tensor("rolled_35")]; tensor clip_34 = clip(alpha = var_22, beta = var_21, x = clip_33)[name = tensor("clip_34")]; - tensor var_2051 = equal(x = encoder__conv_positions, y = clip_34)[name = tensor("op_2051")]; - tensor var_2052_dtype_0 = const()[name = tensor("op_2052_dtype_0"), val = tensor("fp32")]; - tensor var_2053 = const()[name = tensor("op_2053"), val = tensor([1, 16, 1])]; - tensor var_2052 = cast(dtype = var_2052_dtype_0, x = var_2051)[name = tensor("cast_157")]; - tensor fill_mask_35 = reshape(shape = var_2053, x = var_2052)[name = tensor("fill_mask_35")]; + tensor var_1298 = equal(x = encoder__conv_positions, y = clip_34)[name = tensor("op_1298")]; + tensor var_1299_dtype_0 = const()[name = tensor("op_1299_dtype_0"), val = tensor("fp32")]; + tensor var_1300 = const()[name = tensor("op_1300"), val = tensor([1, 16, 1])]; + tensor var_1299 = cast(dtype = var_1299_dtype_0, x = var_1298)[name = tensor("cast_157")]; + tensor fill_mask_35 = reshape(shape = var_1300, x = var_1299)[name = tensor("fill_mask_35")]; tensor x_expanded_35_reps_0 = const()[name = tensor("x_expanded_35_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_35 = tile(reps = x_expanded_35_reps_0, x = var_2043)[name = tensor("x_expanded_35")]; - tensor var_2057 = sub(x = var_18, y = fill_mask_35)[name = tensor("op_2057")]; - tensor var_2058 = mul(x = input_271, y = var_2057)[name = tensor("op_2058")]; - tensor var_2059 = mul(x = x_expanded_35, y = fill_mask_35)[name = tensor("op_2059")]; - tensor filled_window_35 = add(x = var_2058, y = var_2059)[name = tensor("filled_window_35")]; - tensor var_2061 = greater_equal(x = clip_33, y = var_17)[name = tensor("op_2061")]; - tensor var_2062_dtype_0 = const()[name = tensor("op_2062_dtype_0"), val = tensor("fp32")]; - tensor var_2063 = const()[name = tensor("op_2063"), val = tensor([1, 1, 1])]; - tensor var_2062 = cast(dtype = var_2062_dtype_0, x = var_2061)[name = tensor("cast_156")]; - tensor use_roll_35 = reshape(shape = var_2063, x = var_2062)[name = tensor("use_roll_35")]; - tensor var_2065 = sub(x = var_18, y = use_roll_35)[name = tensor("op_2065")]; - tensor var_2066 = mul(x = filled_window_35, y = var_2065)[name = tensor("op_2066")]; - tensor var_2067 = mul(x = rolled_35, y = use_roll_35)[name = tensor("op_2067")]; - tensor input_281 = add(x = var_2066, y = var_2067)[name = tensor("input_281")]; - tensor var_2069 = const()[name = tensor("op_2069"), val = tensor(0x1p+0)]; - tensor var_2070 = add(x = clip_33, y = var_2069)[name = tensor("op_2070")]; - tensor clip_35 = clip(alpha = var_22, beta = var_17, x = var_2070)[name = tensor("clip_35")]; - tensor x_51_axes_0 = const()[name = tensor("x_51_axes_0"), val = tensor([-1])]; - tensor x_51 = layer_norm(axes = x_51_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_281)[name = tensor("x_51")]; - tensor input_283_perm_0 = const()[name = tensor("input_283_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_117_pad_type_0 = const()[name = tensor("inputs_117_pad_type_0"), val = tensor("valid")]; - tensor inputs_117_strides_0 = const()[name = tensor("inputs_117_strides_0"), val = tensor([1])]; - tensor inputs_117_pad_0 = const()[name = tensor("inputs_117_pad_0"), val = tensor([0, 0])]; - tensor inputs_117_dilations_0 = const()[name = tensor("inputs_117_dilations_0"), val = tensor([1])]; - tensor inputs_117_groups_0 = const()[name = tensor("inputs_117_groups_0"), val = tensor(1)]; - tensor input_283 = transpose(perm = input_283_perm_0, x = x_51)[name = tensor("transpose_36")]; - tensor inputs_117 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_117_dilations_0, groups = inputs_117_groups_0, pad = inputs_117_pad_0, pad_type = inputs_117_pad_type_0, strides = inputs_117_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_283)[name = tensor("inputs_117")]; - tensor var_2091_split_sizes_0 = const()[name = tensor("op_2091_split_sizes_0"), val = tensor([256, 256])]; - tensor var_2091_axis_0 = const()[name = tensor("op_2091_axis_0"), val = tensor(1)]; - tensor var_2091_0, tensor var_2091_1 = split(axis = var_2091_axis_0, split_sizes = var_2091_split_sizes_0, x = inputs_117)[name = tensor("op_2091")]; - tensor var_2093 = sigmoid(x = var_2091_1)[name = tensor("op_2093")]; - tensor inputs_119 = mul(x = var_2091_0, y = var_2093)[name = tensor("inputs_119")]; - tensor outputs_aug_35_pad_type_0 = const()[name = tensor("outputs_aug_35_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_35_pad_0 = const()[name = tensor("outputs_aug_35_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_35_groups_0 = const()[name = tensor("outputs_aug_35_groups_0"), val = tensor(256)]; - tensor outputs_aug_35_strides_0 = const()[name = tensor("outputs_aug_35_strides_0"), val = tensor([1])]; - tensor outputs_aug_35_dilations_0 = const()[name = tensor("outputs_aug_35_dilations_0"), val = tensor([1])]; - tensor outputs_aug_35 = conv(dilations = outputs_aug_35_dilations_0, groups = outputs_aug_35_groups_0, pad = outputs_aug_35_pad_0, pad_type = outputs_aug_35_pad_type_0, strides = outputs_aug_35_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_119)[name = tensor("outputs_aug_35")]; - tensor input_285_begin_0 = const()[name = tensor("input_285_begin_0"), val = tensor([0, 0, 0])]; - tensor input_285_end_0 = const()[name = tensor("input_285_end_0"), val = tensor([1, 256, 16])]; - tensor input_285_end_mask_0 = const()[name = tensor("input_285_end_mask_0"), val = tensor([true, true, false])]; - tensor input_285 = slice_by_index(begin = input_285_begin_0, end = input_285_end_0, end_mask = input_285_end_mask_0, x = outputs_aug_35)[name = tensor("input_285")]; - tensor inputs_121 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_285)[name = tensor("inputs_121")]; - tensor input_287 = silu(x = inputs_121)[name = tensor("input_287")]; - tensor input_289_pad_type_0 = const()[name = tensor("input_289_pad_type_0"), val = tensor("valid")]; - tensor input_289_strides_0 = const()[name = tensor("input_289_strides_0"), val = tensor([1])]; - tensor input_289_pad_0 = const()[name = tensor("input_289_pad_0"), val = tensor([0, 0])]; - tensor input_289_dilations_0 = const()[name = tensor("input_289_dilations_0"), val = tensor([1])]; - tensor input_289_groups_0 = const()[name = tensor("input_289_groups_0"), val = tensor(1)]; - tensor input_289 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_289_dilations_0, groups = input_289_groups_0, pad = input_289_pad_0, pad_type = input_289_pad_type_0, strides = input_289_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_287)[name = tensor("input_289")]; - tensor conv_out_35_perm_0 = const()[name = tensor("conv_out_35_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2124_begin_0 = const()[name = tensor("op_2124_begin_0"), val = tensor([0, -1, 0])]; - tensor var_2124_end_0 = const()[name = tensor("op_2124_end_0"), val = tensor([1, 16, 256])]; - tensor var_2124_end_mask_0 = const()[name = tensor("op_2124_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_35 = transpose(perm = conv_out_35_perm_0, x = input_289)[name = tensor("transpose_35")]; - tensor var_2124 = slice_by_index(begin = var_2124_begin_0, end = var_2124_end_0, end_mask = var_2124_end_mask_0, x = conv_out_35)[name = tensor("op_2124")]; - tensor var_2127_begin_0 = const()[name = tensor("op_2127_begin_0"), val = tensor([0, 3, 0])]; - tensor var_2127_end_0 = const()[name = tensor("op_2127_end_0"), val = tensor([1, 4, 256])]; - tensor var_2127_end_mask_0 = const()[name = tensor("op_2127_end_mask_0"), val = tensor([true, false, true])]; - tensor var_2127 = slice_by_index(begin = var_2127_begin_0, end = var_2127_end_0, end_mask = var_2127_end_mask_0, x = x_45)[name = tensor("op_2127")]; - tensor var_2130_begin_0 = const()[name = tensor("op_2130_begin_0"), val = tensor([0, 1, 0])]; - tensor var_2130_end_0 = const()[name = tensor("op_2130_end_0"), val = tensor([1, 16, 256])]; - tensor var_2130_end_mask_0 = const()[name = tensor("op_2130_end_mask_0"), val = tensor([true, true, true])]; - tensor var_2130 = slice_by_index(begin = var_2130_begin_0, end = var_2130_end_0, end_mask = var_2130_end_mask_0, x = input_281)[name = tensor("op_2130")]; + tensor x_expanded_35 = tile(reps = x_expanded_35_reps_0, x = var_1290)[name = tensor("x_expanded_35")]; + tensor var_1304 = sub(x = var_18, y = fill_mask_35)[name = tensor("op_1304")]; + tensor var_1305 = mul(x = window_41, y = var_1304)[name = tensor("op_1305")]; + tensor var_1306 = mul(x = x_expanded_35, y = fill_mask_35)[name = tensor("op_1306")]; + tensor filled_window_35 = add(x = var_1305, y = var_1306)[name = tensor("filled_window_35")]; + tensor var_1308 = greater_equal(x = clip_33, y = var_17)[name = tensor("op_1308")]; + tensor var_1309_dtype_0 = const()[name = tensor("op_1309_dtype_0"), val = tensor("fp32")]; + tensor var_1310 = const()[name = tensor("op_1310"), val = tensor([1, 1, 1])]; + tensor var_1309 = cast(dtype = var_1309_dtype_0, x = var_1308)[name = tensor("cast_156")]; + tensor use_roll_35 = reshape(shape = var_1310, x = var_1309)[name = tensor("use_roll_35")]; + tensor var_1312 = sub(x = var_18, y = use_roll_35)[name = tensor("op_1312")]; + tensor var_1313 = mul(x = filled_window_35, y = var_1312)[name = tensor("op_1313")]; + tensor var_1314 = mul(x = rolled_35, y = use_roll_35)[name = tensor("op_1314")]; + tensor window_43 = add(x = var_1313, y = var_1314)[name = tensor("window_43")]; + tensor var_1316 = const()[name = tensor("op_1316"), val = tensor(0x1p+0)]; + tensor var_1317 = add(x = clip_33, y = var_1316)[name = tensor("op_1317")]; + tensor clip_35 = clip(alpha = var_22, beta = var_17, x = var_1317)[name = tensor("clip_35")]; + tensor var_1320_begin_0 = const()[name = tensor("op_1320_begin_0"), val = tensor([0, 3, 0])]; + tensor var_1320_end_0 = const()[name = tensor("op_1320_end_0"), val = tensor([1, 4, 256])]; + tensor var_1320_end_mask_0 = const()[name = tensor("op_1320_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1320 = slice_by_index(begin = var_1320_begin_0, end = var_1320_end_0, end_mask = var_1320_end_mask_0, x = x_21)[name = tensor("op_1320")]; + tensor var_1323_begin_0 = const()[name = tensor("op_1323_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1323_end_0 = const()[name = tensor("op_1323_end_0"), val = tensor([1, 16, 256])]; + tensor var_1323_end_mask_0 = const()[name = tensor("op_1323_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1323 = slice_by_index(begin = var_1323_begin_0, end = var_1323_end_0, end_mask = var_1323_end_mask_0, x = window_43)[name = tensor("op_1323")]; tensor rolled_37_interleave_0 = const()[name = tensor("rolled_37_interleave_0"), val = tensor(false)]; - tensor rolled_37 = concat(axis = var_34, interleave = rolled_37_interleave_0, values = (var_2130, var_2127))[name = tensor("rolled_37")]; + tensor rolled_37 = concat(axis = var_34, interleave = rolled_37_interleave_0, values = (var_1323, var_1320))[name = tensor("rolled_37")]; tensor clip_36 = clip(alpha = var_22, beta = var_21, x = clip_35)[name = tensor("clip_36")]; - tensor var_2135 = equal(x = encoder__conv_positions, y = clip_36)[name = tensor("op_2135")]; - tensor var_2136_dtype_0 = const()[name = tensor("op_2136_dtype_0"), val = tensor("fp32")]; - tensor var_2137 = const()[name = tensor("op_2137"), val = tensor([1, 16, 1])]; - tensor var_2136 = cast(dtype = var_2136_dtype_0, x = var_2135)[name = tensor("cast_155")]; - tensor fill_mask_37 = reshape(shape = var_2137, x = var_2136)[name = tensor("fill_mask_37")]; + tensor var_1328 = equal(x = encoder__conv_positions, y = clip_36)[name = tensor("op_1328")]; + tensor var_1329_dtype_0 = const()[name = tensor("op_1329_dtype_0"), val = tensor("fp32")]; + tensor var_1330 = const()[name = tensor("op_1330"), val = tensor([1, 16, 1])]; + tensor var_1329 = cast(dtype = var_1329_dtype_0, x = var_1328)[name = tensor("cast_155")]; + tensor fill_mask_37 = reshape(shape = var_1330, x = var_1329)[name = tensor("fill_mask_37")]; tensor x_expanded_37_reps_0 = const()[name = tensor("x_expanded_37_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_37 = tile(reps = x_expanded_37_reps_0, x = var_2127)[name = tensor("x_expanded_37")]; - tensor var_2141 = sub(x = var_18, y = fill_mask_37)[name = tensor("op_2141")]; - tensor var_2142 = mul(x = input_281, y = var_2141)[name = tensor("op_2142")]; - tensor var_2143 = mul(x = x_expanded_37, y = fill_mask_37)[name = tensor("op_2143")]; - tensor filled_window_37 = add(x = var_2142, y = var_2143)[name = tensor("filled_window_37")]; - tensor var_2145 = greater_equal(x = clip_35, y = var_17)[name = tensor("op_2145")]; - tensor var_2146_dtype_0 = const()[name = tensor("op_2146_dtype_0"), val = tensor("fp32")]; - tensor var_2147 = const()[name = tensor("op_2147"), val = tensor([1, 1, 1])]; - tensor var_2146 = cast(dtype = var_2146_dtype_0, x = var_2145)[name = tensor("cast_154")]; - tensor use_roll_37 = reshape(shape = var_2147, x = var_2146)[name = tensor("use_roll_37")]; - tensor var_2149 = sub(x = var_18, y = use_roll_37)[name = tensor("op_2149")]; - tensor var_2150 = mul(x = filled_window_37, y = var_2149)[name = tensor("op_2150")]; - tensor var_2151 = mul(x = rolled_37, y = use_roll_37)[name = tensor("op_2151")]; - tensor input_291 = add(x = var_2150, y = var_2151)[name = tensor("input_291")]; - tensor var_2153 = const()[name = tensor("op_2153"), val = tensor(0x1p+0)]; - tensor var_2154 = add(x = clip_35, y = var_2153)[name = tensor("op_2154")]; - tensor clip_37 = clip(alpha = var_22, beta = var_17, x = var_2154)[name = tensor("clip_37")]; - tensor x_53_axes_0 = const()[name = tensor("x_53_axes_0"), val = tensor([-1])]; - tensor x_53 = layer_norm(axes = x_53_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_291)[name = tensor("x_53")]; - tensor input_293_perm_0 = const()[name = tensor("input_293_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_123_pad_type_0 = const()[name = tensor("inputs_123_pad_type_0"), val = tensor("valid")]; - tensor inputs_123_strides_0 = const()[name = tensor("inputs_123_strides_0"), val = tensor([1])]; - tensor inputs_123_pad_0 = const()[name = tensor("inputs_123_pad_0"), val = tensor([0, 0])]; - tensor inputs_123_dilations_0 = const()[name = tensor("inputs_123_dilations_0"), val = tensor([1])]; - tensor inputs_123_groups_0 = const()[name = tensor("inputs_123_groups_0"), val = tensor(1)]; - tensor input_293 = transpose(perm = input_293_perm_0, x = x_53)[name = tensor("transpose_34")]; - tensor inputs_123 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_123_dilations_0, groups = inputs_123_groups_0, pad = inputs_123_pad_0, pad_type = inputs_123_pad_type_0, strides = inputs_123_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_293)[name = tensor("inputs_123")]; - tensor var_2175_split_sizes_0 = const()[name = tensor("op_2175_split_sizes_0"), val = tensor([256, 256])]; - tensor var_2175_axis_0 = const()[name = tensor("op_2175_axis_0"), val = tensor(1)]; - tensor var_2175_0, tensor var_2175_1 = split(axis = var_2175_axis_0, split_sizes = var_2175_split_sizes_0, x = inputs_123)[name = tensor("op_2175")]; - tensor var_2177 = sigmoid(x = var_2175_1)[name = tensor("op_2177")]; - tensor inputs_125 = mul(x = var_2175_0, y = var_2177)[name = tensor("inputs_125")]; - tensor outputs_aug_37_pad_type_0 = const()[name = tensor("outputs_aug_37_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_37_pad_0 = const()[name = tensor("outputs_aug_37_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_37_groups_0 = const()[name = tensor("outputs_aug_37_groups_0"), val = tensor(256)]; - tensor outputs_aug_37_strides_0 = const()[name = tensor("outputs_aug_37_strides_0"), val = tensor([1])]; - tensor outputs_aug_37_dilations_0 = const()[name = tensor("outputs_aug_37_dilations_0"), val = tensor([1])]; - tensor outputs_aug_37 = conv(dilations = outputs_aug_37_dilations_0, groups = outputs_aug_37_groups_0, pad = outputs_aug_37_pad_0, pad_type = outputs_aug_37_pad_type_0, strides = outputs_aug_37_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_125)[name = tensor("outputs_aug_37")]; - tensor input_295_begin_0 = const()[name = tensor("input_295_begin_0"), val = tensor([0, 0, 0])]; - tensor input_295_end_0 = const()[name = tensor("input_295_end_0"), val = tensor([1, 256, 16])]; - tensor input_295_end_mask_0 = const()[name = tensor("input_295_end_mask_0"), val = tensor([true, true, false])]; - tensor input_295 = slice_by_index(begin = input_295_begin_0, end = input_295_end_0, end_mask = input_295_end_mask_0, x = outputs_aug_37)[name = tensor("input_295")]; - tensor inputs_127 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_295)[name = tensor("inputs_127")]; - tensor input_297 = silu(x = inputs_127)[name = tensor("input_297")]; - tensor input_299_pad_type_0 = const()[name = tensor("input_299_pad_type_0"), val = tensor("valid")]; - tensor input_299_strides_0 = const()[name = tensor("input_299_strides_0"), val = tensor([1])]; - tensor input_299_pad_0 = const()[name = tensor("input_299_pad_0"), val = tensor([0, 0])]; - tensor input_299_dilations_0 = const()[name = tensor("input_299_dilations_0"), val = tensor([1])]; - tensor input_299_groups_0 = const()[name = tensor("input_299_groups_0"), val = tensor(1)]; - tensor input_299 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_299_dilations_0, groups = input_299_groups_0, pad = input_299_pad_0, pad_type = input_299_pad_type_0, strides = input_299_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_297)[name = tensor("input_299")]; - tensor conv_out_37_perm_0 = const()[name = tensor("conv_out_37_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2208_begin_0 = const()[name = tensor("op_2208_begin_0"), val = tensor([0, -1, 0])]; - tensor var_2208_end_0 = const()[name = tensor("op_2208_end_0"), val = tensor([1, 16, 256])]; - tensor var_2208_end_mask_0 = const()[name = tensor("op_2208_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_37 = transpose(perm = conv_out_37_perm_0, x = input_299)[name = tensor("transpose_33")]; - tensor var_2208 = slice_by_index(begin = var_2208_begin_0, end = var_2208_end_0, end_mask = var_2208_end_mask_0, x = conv_out_37)[name = tensor("op_2208")]; - tensor var_2211_begin_0 = const()[name = tensor("op_2211_begin_0"), val = tensor([0, 4, 0])]; - tensor var_2211_end_0 = const()[name = tensor("op_2211_end_0"), val = tensor([1, 1, 256])]; - tensor var_2211_end_mask_0 = const()[name = tensor("op_2211_end_mask_0"), val = tensor([true, true, true])]; - tensor var_2211 = slice_by_index(begin = var_2211_begin_0, end = var_2211_end_0, end_mask = var_2211_end_mask_0, x = x_45)[name = tensor("op_2211")]; - tensor var_2214_begin_0 = const()[name = tensor("op_2214_begin_0"), val = tensor([0, 1, 0])]; - tensor var_2214_end_0 = const()[name = tensor("op_2214_end_0"), val = tensor([1, 16, 256])]; - tensor var_2214_end_mask_0 = const()[name = tensor("op_2214_end_mask_0"), val = tensor([true, true, true])]; - tensor var_2214 = slice_by_index(begin = var_2214_begin_0, end = var_2214_end_0, end_mask = var_2214_end_mask_0, x = input_291)[name = tensor("op_2214")]; + tensor x_expanded_37 = tile(reps = x_expanded_37_reps_0, x = var_1320)[name = tensor("x_expanded_37")]; + tensor var_1334 = sub(x = var_18, y = fill_mask_37)[name = tensor("op_1334")]; + tensor var_1335 = mul(x = window_43, y = var_1334)[name = tensor("op_1335")]; + tensor var_1336 = mul(x = x_expanded_37, y = fill_mask_37)[name = tensor("op_1336")]; + tensor filled_window_37 = add(x = var_1335, y = var_1336)[name = tensor("filled_window_37")]; + tensor var_1338 = greater_equal(x = clip_35, y = var_17)[name = tensor("op_1338")]; + tensor var_1339_dtype_0 = const()[name = tensor("op_1339_dtype_0"), val = tensor("fp32")]; + tensor var_1340 = const()[name = tensor("op_1340"), val = tensor([1, 1, 1])]; + tensor var_1339 = cast(dtype = var_1339_dtype_0, x = var_1338)[name = tensor("cast_154")]; + tensor use_roll_37 = reshape(shape = var_1340, x = var_1339)[name = tensor("use_roll_37")]; + tensor var_1342 = sub(x = var_18, y = use_roll_37)[name = tensor("op_1342")]; + tensor var_1343 = mul(x = filled_window_37, y = var_1342)[name = tensor("op_1343")]; + tensor var_1344 = mul(x = rolled_37, y = use_roll_37)[name = tensor("op_1344")]; + tensor window_45 = add(x = var_1343, y = var_1344)[name = tensor("window_45")]; + tensor var_1346 = const()[name = tensor("op_1346"), val = tensor(0x1p+0)]; + tensor var_1347 = add(x = clip_35, y = var_1346)[name = tensor("op_1347")]; + tensor clip_37 = clip(alpha = var_22, beta = var_17, x = var_1347)[name = tensor("clip_37")]; + tensor var_1350_begin_0 = const()[name = tensor("op_1350_begin_0"), val = tensor([0, 4, 0])]; + tensor var_1350_end_0 = const()[name = tensor("op_1350_end_0"), val = tensor([1, 1, 256])]; + tensor var_1350_end_mask_0 = const()[name = tensor("op_1350_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1350 = slice_by_index(begin = var_1350_begin_0, end = var_1350_end_0, end_mask = var_1350_end_mask_0, x = x_21)[name = tensor("op_1350")]; + tensor var_1353_begin_0 = const()[name = tensor("op_1353_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1353_end_0 = const()[name = tensor("op_1353_end_0"), val = tensor([1, 16, 256])]; + tensor var_1353_end_mask_0 = const()[name = tensor("op_1353_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1353 = slice_by_index(begin = var_1353_begin_0, end = var_1353_end_0, end_mask = var_1353_end_mask_0, x = window_45)[name = tensor("op_1353")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_34, interleave = rolled_interleave_0, values = (var_2214, var_2211))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_34, interleave = rolled_interleave_0, values = (var_1353, var_1350))[name = tensor("rolled")]; tensor clip_38 = clip(alpha = var_22, beta = var_21, x = clip_37)[name = tensor("clip_38")]; - tensor var_2219 = equal(x = encoder__conv_positions, y = clip_38)[name = tensor("op_2219")]; - tensor var_2220_dtype_0 = const()[name = tensor("op_2220_dtype_0"), val = tensor("fp32")]; - tensor var_2221 = const()[name = tensor("op_2221"), val = tensor([1, 16, 1])]; - tensor var_2220 = cast(dtype = var_2220_dtype_0, x = var_2219)[name = tensor("cast_153")]; - tensor fill_mask = reshape(shape = var_2221, x = var_2220)[name = tensor("fill_mask")]; + tensor var_1358 = equal(x = encoder__conv_positions, y = clip_38)[name = tensor("op_1358")]; + tensor var_1359_dtype_0 = const()[name = tensor("op_1359_dtype_0"), val = tensor("fp32")]; + tensor var_1360 = const()[name = tensor("op_1360"), val = tensor([1, 16, 1])]; + tensor var_1359 = cast(dtype = var_1359_dtype_0, x = var_1358)[name = tensor("cast_153")]; + tensor fill_mask = reshape(shape = var_1360, x = var_1359)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_2211)[name = tensor("x_expanded")]; - tensor var_2225 = sub(x = var_18, y = fill_mask)[name = tensor("op_2225")]; - tensor var_2226 = mul(x = input_291, y = var_2225)[name = tensor("op_2226")]; - tensor var_2227 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_2227")]; - tensor filled_window = add(x = var_2226, y = var_2227)[name = tensor("filled_window")]; - tensor var_2229 = greater_equal(x = clip_37, y = var_17)[name = tensor("op_2229")]; - tensor var_2230_dtype_0 = const()[name = tensor("op_2230_dtype_0"), val = tensor("fp32")]; - tensor var_2231 = const()[name = tensor("op_2231"), val = tensor([1, 1, 1])]; - tensor var_2230 = cast(dtype = var_2230_dtype_0, x = var_2229)[name = tensor("cast_152")]; - tensor use_roll = reshape(shape = var_2231, x = var_2230)[name = tensor("use_roll")]; - tensor var_2233 = sub(x = var_18, y = use_roll)[name = tensor("op_2233")]; - tensor var_2234 = mul(x = filled_window, y = var_2233)[name = tensor("op_2234")]; - tensor var_2235 = mul(x = rolled, y = use_roll)[name = tensor("op_2235")]; - tensor input_301 = add(x = var_2234, y = var_2235)[name = tensor("input_301")]; - tensor var_2237 = const()[name = tensor("op_2237"), val = tensor(0x1p+0)]; - tensor var_2238 = add(x = clip_37, y = var_2237)[name = tensor("op_2238")]; - tensor clip_39 = clip(alpha = var_22, beta = var_17, x = var_2238)[name = tensor("clip_39")]; - tensor x_55_axes_0 = const()[name = tensor("x_55_axes_0"), val = tensor([-1])]; - tensor x_55 = layer_norm(axes = x_55_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_301)[name = tensor("x_55")]; - tensor input_303_perm_0 = const()[name = tensor("input_303_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_129_pad_type_0 = const()[name = tensor("inputs_129_pad_type_0"), val = tensor("valid")]; - tensor inputs_129_strides_0 = const()[name = tensor("inputs_129_strides_0"), val = tensor([1])]; - tensor inputs_129_pad_0 = const()[name = tensor("inputs_129_pad_0"), val = tensor([0, 0])]; - tensor inputs_129_dilations_0 = const()[name = tensor("inputs_129_dilations_0"), val = tensor([1])]; - tensor inputs_129_groups_0 = const()[name = tensor("inputs_129_groups_0"), val = tensor(1)]; - tensor input_303 = transpose(perm = input_303_perm_0, x = x_55)[name = tensor("transpose_32")]; - tensor inputs_129 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_129_dilations_0, groups = inputs_129_groups_0, pad = inputs_129_pad_0, pad_type = inputs_129_pad_type_0, strides = inputs_129_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_303)[name = tensor("inputs_129")]; - tensor var_2259_split_sizes_0 = const()[name = tensor("op_2259_split_sizes_0"), val = tensor([256, 256])]; - tensor var_2259_axis_0 = const()[name = tensor("op_2259_axis_0"), val = tensor(1)]; - tensor var_2259_0, tensor var_2259_1 = split(axis = var_2259_axis_0, split_sizes = var_2259_split_sizes_0, x = inputs_129)[name = tensor("op_2259")]; - tensor var_2261 = sigmoid(x = var_2259_1)[name = tensor("op_2261")]; - tensor inputs_131 = mul(x = var_2259_0, y = var_2261)[name = tensor("inputs_131")]; + tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1350)[name = tensor("x_expanded")]; + tensor var_1364 = sub(x = var_18, y = fill_mask)[name = tensor("op_1364")]; + tensor var_1365 = mul(x = window_45, y = var_1364)[name = tensor("op_1365")]; + tensor var_1366 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1366")]; + tensor filled_window = add(x = var_1365, y = var_1366)[name = tensor("filled_window")]; + tensor var_1368 = greater_equal(x = clip_37, y = var_17)[name = tensor("op_1368")]; + tensor var_1369_dtype_0 = const()[name = tensor("op_1369_dtype_0"), val = tensor("fp32")]; + tensor var_1370 = const()[name = tensor("op_1370"), val = tensor([1, 1, 1])]; + tensor var_1369 = cast(dtype = var_1369_dtype_0, x = var_1368)[name = tensor("cast_152")]; + tensor use_roll = reshape(shape = var_1370, x = var_1369)[name = tensor("use_roll")]; + tensor var_1372 = sub(x = var_18, y = use_roll)[name = tensor("op_1372")]; + tensor var_1373 = mul(x = filled_window, y = var_1372)[name = tensor("op_1373")]; + tensor var_1374 = mul(x = rolled, y = use_roll)[name = tensor("op_1374")]; + tensor window = add(x = var_1373, y = var_1374)[name = tensor("window")]; + tensor var_1376 = const()[name = tensor("op_1376"), val = tensor(0x1p+0)]; + tensor var_1377 = add(x = clip_37, y = var_1376)[name = tensor("op_1377")]; + tensor clip_39 = clip(alpha = var_22, beta = var_17, x = var_1377)[name = tensor("clip_39")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_31, interleave = input_141_interleave_0, values = (window_39, window_41, window_43, window_45, window))[name = tensor("input_141")]; + tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; + tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; + tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_33_pad_type_0 = const()[name = tensor("inputs_33_pad_type_0"), val = tensor("valid")]; + tensor inputs_33_strides_0 = const()[name = tensor("inputs_33_strides_0"), val = tensor([1])]; + tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; + tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; + tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; + tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; + tensor var_1400_split_sizes_0 = const()[name = tensor("op_1400_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1400_axis_0 = const()[name = tensor("op_1400_axis_0"), val = tensor(1)]; + tensor var_1400_0, tensor var_1400_1 = split(axis = var_1400_axis_0, split_sizes = var_1400_split_sizes_0, x = inputs_33)[name = tensor("op_1400")]; + tensor var_1402 = sigmoid(x = var_1400_1)[name = tensor("op_1402")]; + tensor inputs_35 = mul(x = var_1400_0, y = var_1402)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; tensor outputs_aug_strides_0 = const()[name = tensor("outputs_aug_strides_0"), val = tensor([1])]; tensor outputs_aug_dilations_0 = const()[name = tensor("outputs_aug_dilations_0"), val = tensor([1])]; - tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_131)[name = tensor("outputs_aug")]; - tensor input_305_begin_0 = const()[name = tensor("input_305_begin_0"), val = tensor([0, 0, 0])]; - tensor input_305_end_0 = const()[name = tensor("input_305_end_0"), val = tensor([1, 256, 16])]; - tensor input_305_end_mask_0 = const()[name = tensor("input_305_end_mask_0"), val = tensor([true, true, false])]; - tensor input_305 = slice_by_index(begin = input_305_begin_0, end = input_305_end_0, end_mask = input_305_end_mask_0, x = outputs_aug)[name = tensor("input_305")]; - tensor inputs_133 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_305)[name = tensor("inputs_133")]; - tensor input_307 = silu(x = inputs_133)[name = tensor("input_307")]; - tensor input_309_pad_type_0 = const()[name = tensor("input_309_pad_type_0"), val = tensor("valid")]; - tensor input_309_strides_0 = const()[name = tensor("input_309_strides_0"), val = tensor([1])]; - tensor input_309_pad_0 = const()[name = tensor("input_309_pad_0"), val = tensor([0, 0])]; - tensor input_309_dilations_0 = const()[name = tensor("input_309_dilations_0"), val = tensor([1])]; - tensor input_309_groups_0 = const()[name = tensor("input_309_groups_0"), val = tensor(1)]; - tensor input_309 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_309_dilations_0, groups = input_309_groups_0, pad = input_309_pad_0, pad_type = input_309_pad_type_0, strides = input_309_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_307)[name = tensor("input_309")]; - tensor conv_out_39_perm_0 = const()[name = tensor("conv_out_39_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2292_begin_0 = const()[name = tensor("op_2292_begin_0"), val = tensor([0, -1, 0])]; - tensor var_2292_end_0 = const()[name = tensor("op_2292_end_0"), val = tensor([1, 16, 256])]; - tensor var_2292_end_mask_0 = const()[name = tensor("op_2292_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_39 = transpose(perm = conv_out_39_perm_0, x = input_309)[name = tensor("transpose_31")]; - tensor var_2292 = slice_by_index(begin = var_2292_begin_0, end = var_2292_end_0, end_mask = var_2292_end_mask_0, x = conv_out_39)[name = tensor("op_2292")]; - tensor var_2295_interleave_0 = const()[name = tensor("op_2295_interleave_0"), val = tensor(false)]; - tensor var_2295 = concat(axis = var_34, interleave = var_2295_interleave_0, values = (var_1956, var_2040, var_2124, var_2208, var_2292))[name = tensor("op_2295")]; - tensor input_311 = add(x = x_45, y = var_2295)[name = tensor("input_311")]; - tensor input_313_axes_0 = const()[name = tensor("input_313_axes_0"), val = tensor([-1])]; - tensor input_313 = layer_norm(axes = input_313_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_311)[name = tensor("input_313")]; - tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_313)[name = tensor("linear_35")]; - tensor input_315 = silu(x = inputs)[name = tensor("input_315")]; - tensor input_319 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_315)[name = tensor("linear_36")]; - tensor var_2318 = const()[name = tensor("op_2318"), val = tensor(0x1p-1)]; - tensor var_2319 = mul(x = input_319, y = var_2318)[name = tensor("op_2319")]; - tensor input_321 = add(x = var_2319, y = input_311)[name = tensor("input_321")]; - tensor x_57_axes_0 = const()[name = tensor("x_57_axes_0"), val = tensor([-1])]; - tensor x_57 = layer_norm(axes = x_57_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_36, gamma = encoder_layer_norm_3_weight, x = input_321)[name = tensor("x_57")]; + tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_35)[name = tensor("outputs_aug")]; + tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; + tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([5, 256, 16])]; + tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; + tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug)[name = tensor("input_145")]; + tensor inputs_37 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_145)[name = tensor("inputs_37")]; + tensor input_147 = silu(x = inputs_37)[name = tensor("input_147")]; + tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; + tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; + tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; + tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; + tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; + tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; + tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1433_begin_0 = const()[name = tensor("op_1433_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1433_end_0 = const()[name = tensor("op_1433_end_0"), val = tensor([5, 16, 256])]; + tensor var_1433_end_mask_0 = const()[name = tensor("op_1433_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_1433 = slice_by_index(begin = var_1433_begin_0, end = var_1433_end_0, end_mask = var_1433_end_mask_0, x = conv_out_7)[name = tensor("op_1433")]; + tensor var_1435_perm_0 = const()[name = tensor("op_1435_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1435 = transpose(perm = var_1435_perm_0, x = var_1433)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_1435)[name = tensor("input_151")]; + tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; + tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; + tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; + tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; + tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; + tensor var_1458 = const()[name = tensor("op_1458"), val = tensor(0x1p-1)]; + tensor var_1459 = mul(x = input_159, y = var_1458)[name = tensor("op_1459")]; + tensor input_161 = add(x = var_1459, y = input_151)[name = tensor("input_161")]; + tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; + tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_36, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; tensor cat_interleave_0 = const()[name = tensor("cat_interleave_0"), val = tensor(false)]; - tensor x_ct = transpose(perm = x_ct_perm_0, x = x_57)[name = tensor("transpose_30")]; + tensor x_ct = transpose(perm = x_ct_perm_0, x = x_25)[name = tensor("transpose_30")]; tensor cat = concat(axis = var_28, interleave = cat_interleave_0, values = (cnn_window, x_ct))[name = tensor("cat")]; tensor conv_out_pad_type_0 = const()[name = tensor("conv_out_pad_type_0"), val = tensor("valid")]; tensor conv_out_strides_0 = const()[name = tensor("conv_out_strides_0"), val = tensor([1])]; @@ -2097,44 +1482,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_2337_begin_0 = const()[name = tensor("op_2337_begin_0"), val = tensor([0, 0, 5])]; - tensor var_2337_end_0 = const()[name = tensor("op_2337_end_0"), val = tensor([1, 256, 23])]; - tensor var_2337_end_mask_0 = const()[name = tensor("op_2337_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_2337_begin_0, end = var_2337_end_0, end_mask = var_2337_end_mask_0, x = cat)[name = tensor("op_2337")]; - tensor input_323_perm_0 = const()[name = tensor("input_323_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2339 = const()[name = tensor("op_2339"), val = tensor([-1])]; - tensor input_323 = transpose(perm = input_323_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_2340 = reduce_l2_norm(axes = var_2339, keep_dims = var_37, x = input_323)[name = tensor("op_2340")]; - tensor const_28 = const()[name = tensor("const_28"), val = tensor(0x1.fffffep+127)]; - tensor clip_40 = clip(alpha = var_12, beta = const_28, x = var_2340)[name = tensor("clip_40")]; - tensor emb = real_div(x = input_323, y = clip_40)[name = tensor("emb")]; - tensor var_2344_axis_0 = const()[name = tensor("op_2344_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_2344_axis_0, values = (var_215, var_763, var_1311, nkv_1))[name = tensor("op_2344")]; - tensor var_2346_axis_0 = const()[name = tensor("op_2346_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_2346_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_2346")]; - tensor var_2348_axis_0 = const()[name = tensor("op_2348_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_2348_axis_0, values = (input_61, input_141, input_221, input_301))[name = tensor("op_2348")]; - tensor var_2350_axis_0 = const()[name = tensor("op_2350_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_2350_axis_0, values = (clip_9, clip_19, clip_29, clip_39))[name = tensor("op_2350")]; - tensor var_2360 = const()[name = tensor("op_2360"), val = tensor(0x1.5798eep-27)]; - tensor var_2366 = const()[name = tensor("op_2366"), val = tensor(0x1.4f8b58p-17)]; - tensor var_2368 = const()[name = tensor("op_2368"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_2369 = const()[name = tensor("op_2369"), val = tensor(true)]; - tensor var_2371 = const()[name = tensor("op_2371"), val = tensor(0x1p+0)]; - tensor var_2375 = const()[name = tensor("op_2375"), val = tensor(-1)]; - tensor var_2380 = const()[name = tensor("op_2380"), val = tensor(0)]; + tensor var_1477_begin_0 = const()[name = tensor("op_1477_begin_0"), val = tensor([0, 0, 5])]; + tensor var_1477_end_0 = const()[name = tensor("op_1477_end_0"), val = tensor([1, 256, 23])]; + tensor var_1477_end_mask_0 = const()[name = tensor("op_1477_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_1477_begin_0, end = var_1477_end_0, end_mask = var_1477_end_mask_0, x = cat)[name = tensor("op_1477")]; + tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1479 = const()[name = tensor("op_1479"), val = tensor([-1])]; + tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; + tensor var_1480 = reduce_l2_norm(axes = var_1479, keep_dims = var_37, x = input_163)[name = tensor("op_1480")]; + tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; + tensor clip_40 = clip(alpha = var_12, beta = const_12, x = var_1480)[name = tensor("clip_40")]; + tensor emb = real_div(x = input_163, y = clip_40)[name = tensor("emb")]; + tensor var_1484_axis_0 = const()[name = tensor("op_1484_axis_0"), val = tensor(0)]; + tensor enc_kv_new = stack(axis = var_1484_axis_0, values = (var_215, var_548, var_881, nkv_1))[name = tensor("op_1484")]; + tensor var_1486_axis_0 = const()[name = tensor("op_1486_axis_0"), val = tensor(0)]; + tensor enc_scale_new = stack(axis = var_1486_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1486")]; + tensor var_1488_axis_0 = const()[name = tensor("op_1488_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1488_axis_0, values = (window_11, window_23, window_35, window))[name = tensor("op_1488")]; + tensor var_1490_axis_0 = const()[name = tensor("op_1490_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1490_axis_0, values = (clip_9, clip_19, clip_29, clip_39))[name = tensor("op_1490")]; + tensor var_1500 = const()[name = tensor("op_1500"), val = tensor(0x1.5798eep-27)]; + tensor var_1505 = const()[name = tensor("op_1505"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1507 = const()[name = tensor("op_1507"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1508 = const()[name = tensor("op_1508"), val = tensor(true)]; + tensor var_1510 = const()[name = tensor("op_1510"), val = tensor(0x1p+0)]; + tensor var_1514 = const()[name = tensor("op_1514"), val = tensor(-1)]; + tensor var_1520 = const()[name = tensor("op_1520"), val = tensor(0)]; tensor pos = const()[name = tensor("pos"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395840)))]; - tensor var_2442_axes_0 = const()[name = tensor("op_2442_axes_0"), val = tensor([2])]; - tensor var_2442 = expand_dims(axes = var_2442_axes_0, x = emb)[name = tensor("op_2442")]; - tensor var_2443 = const()[name = tensor("op_2443"), val = tensor([1, 1, 6, 1])]; - tensor emb_exp = tile(reps = var_2443, x = var_2442)[name = tensor("emb_exp")]; - tensor input_325_interleave_0 = const()[name = tensor("input_325_interleave_0"), val = tensor(false)]; - tensor input_325 = concat(axis = var_2375, interleave = input_325_interleave_0, values = (emb_exp, pos))[name = tensor("input_325")]; - tensor x_59 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_325)[name = tensor("linear_37")]; - tensor var_2450_perm_0 = const()[name = tensor("op_2450_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2454 = const()[name = tensor("op_2454"), val = tensor([6, 5, 256])]; - tensor var_2450 = transpose(perm = var_2450_perm_0, x = x_59)[name = tensor("transpose_28")]; - tensor x_61 = reshape(shape = var_2454, x = var_2450)[name = tensor("x_61")]; + tensor var_1582_axes_0 = const()[name = tensor("op_1582_axes_0"), val = tensor([2])]; + tensor var_1582 = expand_dims(axes = var_1582_axes_0, x = emb)[name = tensor("op_1582")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 6, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1582)[name = tensor("emb_exp")]; + tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; + tensor input_165 = concat(axis = var_1514, interleave = input_165_interleave_0, values = (emb_exp, pos))[name = tensor("input_165")]; + tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; + tensor var_1590_perm_0 = const()[name = tensor("op_1590_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1594 = const()[name = tensor("op_1594"), val = tensor([6, 5, 256])]; + tensor var_1590 = transpose(perm = var_1590_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1594, x = var_1590)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 6, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -2145,132 +1530,132 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_2462 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_61)[name = tensor("linear_38")]; - tensor var_2463 = const()[name = tensor("op_2463"), val = tensor([6, 5, 4, 64])]; - tensor var_2464 = reshape(shape = var_2463, x = var_2462)[name = tensor("op_2464")]; + tensor var_1602 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1603 = const()[name = tensor("op_1603"), val = tensor([6, 5, 4, 64])]; + tensor var_1604 = reshape(shape = var_1603, x = var_1602)[name = tensor("op_1604")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2468 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_61)[name = tensor("linear_39")]; - tensor var_2469 = const()[name = tensor("op_2469"), val = tensor(0x1p-3)]; - tensor var_2470 = mul(x = var_2468, y = var_2469)[name = tensor("op_2470")]; - tensor var_2471 = const()[name = tensor("op_2471"), val = tensor([6, 5, 4, 64])]; - tensor var_2472 = reshape(shape = var_2471, x = var_2470)[name = tensor("op_2472")]; + tensor var_1608 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1609 = const()[name = tensor("op_1609"), val = tensor(0x1p-3)]; + tensor var_1610 = mul(x = var_1608, y = var_1609)[name = tensor("op_1610")]; + tensor var_1611 = const()[name = tensor("op_1611"), val = tensor([6, 5, 4, 64])]; + tensor var_1612 = reshape(shape = var_1611, x = var_1610)[name = tensor("op_1612")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2476 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_61)[name = tensor("linear_40")]; - tensor var_2477 = const()[name = tensor("op_2477"), val = tensor([6, 5, 4, 64])]; - tensor var_2478 = reshape(shape = var_2477, x = var_2476)[name = tensor("op_2478")]; + tensor var_1616 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1617 = const()[name = tensor("op_1617"), val = tensor([6, 5, 4, 64])]; + tensor var_1618 = reshape(shape = var_1617, x = var_1616)[name = tensor("op_1618")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_329 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_61)[name = tensor("linear_41")]; + tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_2380, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1520, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; - tensor const_36 = const()[name = tensor("const_36"), val = tensor(0x1.fffffep+127)]; - tensor clip_41 = clip(alpha = var_2371, beta = const_36, x = s_t_9)[name = tensor("clip_41")]; + tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; + tensor clip_41 = clip(alpha = var_1510, beta = const_20, x = s_t_9)[name = tensor("clip_41")]; tensor sqrt_s_t_9 = sqrt(x = clip_41)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_2472)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_2464)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1612)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1604)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_2490 = const()[name = tensor("op_2490"), val = tensor([1, 5])]; - tensor var_2491 = reshape(shape = var_2490, x = valid_mask)[name = tensor("op_2491")]; - tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_2491)[name = tensor("causal_with_valid_1")]; - tensor var_2493 = const()[name = tensor("op_2493"), val = tensor([5, 1])]; - tensor var_2494 = reshape(shape = var_2493, x = sqrt_s_t_9)[name = tensor("op_2494")]; - tensor M_9 = real_div(x = causal_with_valid_1, y = var_2494)[name = tensor("M_9")]; - tensor var_2496 = mul(x = qk_9, y = M_9)[name = tensor("op_2496")]; + tensor var_1630 = const()[name = tensor("op_1630"), val = tensor([1, 5])]; + tensor var_1631 = reshape(shape = var_1630, x = valid_mask)[name = tensor("op_1631")]; + tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1631)[name = tensor("causal_with_valid_1")]; + tensor var_1633 = const()[name = tensor("op_1633"), val = tensor([5, 1])]; + tensor var_1634 = reshape(shape = var_1633, x = sqrt_s_t_9)[name = tensor("op_1634")]; + tensor M_9 = real_div(x = causal_with_valid_1, y = var_1634)[name = tensor("M_9")]; + tensor var_1636 = mul(x = qk_9, y = M_9)[name = tensor("op_1636")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_2478)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_2496, y = v_9)[name = tensor("inner_9")]; - tensor var_2498_transpose_x_0 = const()[name = tensor("op_2498_transpose_x_0"), val = tensor(false)]; - tensor var_2498_transpose_y_0 = const()[name = tensor("op_2498_transpose_y_0"), val = tensor(false)]; - tensor var_2498 = matmul(transpose_x = var_2498_transpose_x_0, transpose_y = var_2498_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_2498")]; - tensor var_2499 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_2499")]; - tensor var_2500 = const()[name = tensor("op_2500"), val = tensor([1, 1, 5, 1])]; - tensor var_2501 = reshape(shape = var_2500, x = var_2499)[name = tensor("op_2501")]; - tensor cross_9 = mul(x = var_2498, y = var_2501)[name = tensor("cross_9")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1618)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1636, y = v_9)[name = tensor("inner_9")]; + tensor var_1638_transpose_x_0 = const()[name = tensor("op_1638_transpose_x_0"), val = tensor(false)]; + tensor var_1638_transpose_y_0 = const()[name = tensor("op_1638_transpose_y_0"), val = tensor(false)]; + tensor var_1638 = matmul(transpose_x = var_1638_transpose_x_0, transpose_y = var_1638_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1638")]; + tensor var_1639 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1639")]; + tensor var_1640 = const()[name = tensor("op_1640"), val = tensor([1, 1, 5, 1])]; + tensor var_1641 = reshape(shape = var_1640, x = var_1639)[name = tensor("op_1641")]; + tensor cross_9 = mul(x = var_1638, y = var_1641)[name = tensor("cross_9")]; tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; - tensor var_2504 = const()[name = tensor("op_2504"), val = tensor([1, 1, 5, 1])]; - tensor var_2505 = reshape(shape = var_2504, x = valid_mask)[name = tensor("op_2505")]; - tensor v_masked_1 = mul(x = v_9, y = var_2505)[name = tensor("v_masked_1")]; - tensor var_2507 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_2507")]; - tensor var_2509_transpose_x_1 = const()[name = tensor("op_2509_transpose_x_1"), val = tensor(true)]; - tensor var_2509_transpose_y_1 = const()[name = tensor("op_2509_transpose_y_1"), val = tensor(false)]; - tensor var_2509 = matmul(transpose_x = var_2509_transpose_x_1, transpose_y = var_2509_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_2509")]; - tensor new_kv_unnorm_9 = add(x = var_2507, y = var_2509)[name = tensor("new_kv_unnorm_9")]; - tensor var_2511_keep_dims_0 = const()[name = tensor("op_2511_keep_dims_0"), val = tensor(false)]; - tensor var_2511 = reduce_sum(keep_dims = var_2511_keep_dims_0, x = valid_mask)[name = tensor("op_2511")]; - tensor var_2512 = const()[name = tensor("op_2512"), val = tensor([1])]; - tensor var_2513 = reshape(shape = var_2512, x = var_2511)[name = tensor("op_2513")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_2513)[name = tensor("new_scale_9")]; - tensor const_37 = const()[name = tensor("const_37"), val = tensor(0x1.fffffep+127)]; - tensor clip_42 = clip(alpha = var_2371, beta = const_37, x = new_scale_9)[name = tensor("clip_42")]; + tensor var_1644 = const()[name = tensor("op_1644"), val = tensor([1, 1, 5, 1])]; + tensor var_1645 = reshape(shape = var_1644, x = valid_mask)[name = tensor("op_1645")]; + tensor v_masked_1 = mul(x = v_9, y = var_1645)[name = tensor("v_masked_1")]; + tensor var_1647 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1647")]; + tensor var_1649_transpose_x_1 = const()[name = tensor("op_1649_transpose_x_1"), val = tensor(true)]; + tensor var_1649_transpose_y_1 = const()[name = tensor("op_1649_transpose_y_1"), val = tensor(false)]; + tensor var_1649 = matmul(transpose_x = var_1649_transpose_x_1, transpose_y = var_1649_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1649")]; + tensor new_kv_unnorm_9 = add(x = var_1647, y = var_1649)[name = tensor("new_kv_unnorm_9")]; + tensor var_1651_keep_dims_0 = const()[name = tensor("op_1651_keep_dims_0"), val = tensor(false)]; + tensor var_1651 = reduce_sum(keep_dims = var_1651_keep_dims_0, x = valid_mask)[name = tensor("op_1651")]; + tensor var_1652 = const()[name = tensor("op_1652"), val = tensor([1])]; + tensor var_1653 = reshape(shape = var_1652, x = var_1651)[name = tensor("op_1653")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1653)[name = tensor("new_scale_9")]; + tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; + tensor clip_42 = clip(alpha = var_1510, beta = const_21, x = new_scale_9)[name = tensor("clip_42")]; tensor sqrt_new_scale_1 = sqrt(x = clip_42)[name = tensor("sqrt_new_scale_1")]; - tensor var_2517 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_2517")]; - tensor var_2518_perm_0 = const()[name = tensor("op_2518_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1657 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1657")]; + tensor var_1658_perm_0 = const()[name = tensor("op_1658_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_2518 = transpose(perm = var_2518_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_2368, x = var_2518)[name = tensor("out_27")]; - tensor var_2522 = const()[name = tensor("op_2522"), val = tensor([6, 5, 256])]; - tensor out_29 = reshape(shape = var_2522, x = out_27)[name = tensor("out_29")]; - tensor var_2524 = silu(x = input_329)[name = tensor("op_2524")]; - tensor input_331 = mul(x = var_2524, y = out_29)[name = tensor("input_331")]; - tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_331)[name = tensor("linear_42")]; - tensor input_333 = add(x = x_61, y = ret_out_9)[name = tensor("input_333")]; + tensor var_1658 = transpose(perm = var_1658_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1507, x = var_1658)[name = tensor("out_27")]; + tensor var_1662 = const()[name = tensor("op_1662"), val = tensor([6, 5, 256])]; + tensor out_29 = reshape(shape = var_1662, x = out_27)[name = tensor("out_29")]; + tensor var_1664 = silu(x = input_169)[name = tensor("op_1664")]; + tensor input_171 = mul(x = var_1664, y = out_29)[name = tensor("input_171")]; + tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; + tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_2366, gamma = decoder_norm11_0_weight, x = input_333)[name = tensor("xt_1")]; - tensor var_2534 = const()[name = tensor("op_2534"), val = tensor([1, 6, 5, 256])]; - tensor var_2535 = reshape(shape = var_2534, x = xt_1)[name = tensor("op_2535")]; - tensor var_2536_perm_0 = const()[name = tensor("op_2536_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2539 = const()[name = tensor("op_2539"), val = tensor([5, 6, 256])]; - tensor var_2536 = transpose(perm = var_2536_perm_0, x = var_2535)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_2539, x = var_2536)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1505, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1674 = const()[name = tensor("op_1674"), val = tensor([1, 6, 5, 256])]; + tensor var_1675 = reshape(shape = var_1674, x = xt_1)[name = tensor("op_1675")]; + tensor var_1676_perm_0 = const()[name = tensor("op_1676_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1679 = const()[name = tensor("op_1679"), val = tensor([5, 6, 256])]; + tensor var_1676 = transpose(perm = var_1676_perm_0, x = var_1675)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1679, x = var_1676)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_2562 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_20 = const()[name = tensor("concat_20"), val = tensor([6, 5, 3, 256])]; - tensor var_2564 = reshape(shape = concat_20, x = var_2562)[name = tensor("op_2564")]; - tensor var_2565_axes_0 = const()[name = tensor("op_2565_axes_0"), val = tensor([0])]; - tensor var_2565 = expand_dims(axes = var_2565_axes_0, x = var_2564)[name = tensor("op_2565")]; - tensor var_2566_perm_0 = const()[name = tensor("op_2566_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2567_axes_0 = const()[name = tensor("op_2567_axes_0"), val = tensor([-2])]; - tensor var_2566 = transpose(perm = var_2566_perm_0, x = var_2565)[name = tensor("transpose_21")]; - tensor var_2567 = squeeze(axes = var_2567_axes_0, x = var_2566)[name = tensor("op_2567")]; + tensor var_1702 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_21 = const()[name = tensor("concat_21"), val = tensor([6, 5, 3, 256])]; + tensor var_1704 = reshape(shape = concat_21, x = var_1702)[name = tensor("op_1704")]; + tensor var_1705_axes_0 = const()[name = tensor("op_1705_axes_0"), val = tensor([0])]; + tensor var_1705 = expand_dims(axes = var_1705_axes_0, x = var_1704)[name = tensor("op_1705")]; + tensor var_1706_perm_0 = const()[name = tensor("op_1706_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1707_axes_0 = const()[name = tensor("op_1707_axes_0"), val = tensor([-2])]; + tensor var_1706 = transpose(perm = var_1706_perm_0, x = var_1705)[name = tensor("transpose_21")]; + tensor var_1707 = squeeze(axes = var_1707_axes_0, x = var_1706)[name = tensor("op_1707")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 6, 5, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_2567)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1707)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 6, 5, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_2567)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1707)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 6, 5, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_2567)[name = tensor("v_11")]; - tensor var_2575 = const()[name = tensor("op_2575"), val = tensor([6, 20, 64])]; - tensor var_2576 = reshape(shape = var_2575, x = q_11)[name = tensor("op_2576")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1707)[name = tensor("v_11")]; + tensor var_1715 = const()[name = tensor("op_1715"), val = tensor([6, 20, 64])]; + tensor var_1716 = reshape(shape = var_1715, x = q_11)[name = tensor("op_1716")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2582 = const()[name = tensor("op_2582"), val = tensor([6, 20, 64])]; - tensor var_2583 = reshape(shape = var_2582, x = k_11)[name = tensor("op_2583")]; + tensor var_1722 = const()[name = tensor("op_1722"), val = tensor([6, 20, 64])]; + tensor var_1723 = reshape(shape = var_1722, x = k_11)[name = tensor("op_1723")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2589 = const()[name = tensor("op_2589"), val = tensor([6, 20, 64])]; - tensor var_2590 = reshape(shape = var_2589, x = v_11)[name = tensor("op_2590")]; + tensor var_1729 = const()[name = tensor("op_1729"), val = tensor([6, 20, 64])]; + tensor var_1730 = reshape(shape = var_1729, x = v_11)[name = tensor("op_1730")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2593 = const()[name = tensor("op_2593"), val = tensor([5, 4, 6, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_2576)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_2593, x = q_13)[name = tensor("q_15")]; - tensor var_2595 = const()[name = tensor("op_2595"), val = tensor([5, 4, 6, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_2583)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_2595, x = k_13)[name = tensor("k_15")]; - tensor var_2597 = const()[name = tensor("op_2597"), val = tensor([5, 4, 6, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_2590)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_2597, x = v_13)[name = tensor("v_15")]; + tensor var_1733 = const()[name = tensor("op_1733"), val = tensor([5, 4, 6, 64])]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1716)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1733, x = q_13)[name = tensor("q_15")]; + tensor var_1735 = const()[name = tensor("op_1735"), val = tensor([5, 4, 6, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1723)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1735, x = k_13)[name = tensor("k_15")]; + tensor var_1737 = const()[name = tensor("op_1737"), val = tensor([5, 4, 6, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1730)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1737, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -2281,30 +1666,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_2600 = const()[name = tensor("op_2600"), val = tensor([2, 0, 1, 3])]; - tensor var_2605 = const()[name = tensor("op_2605"), val = tensor([30, 256])]; - tensor var_2601 = transpose(perm = var_2600, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_2605, x = var_2601)[name = tensor("attn_output_3")]; + tensor var_1740 = const()[name = tensor("op_1740"), val = tensor([2, 0, 1, 3])]; + tensor var_1745 = const()[name = tensor("op_1745"), val = tensor([30, 256])]; + tensor var_1741 = transpose(perm = var_1740, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1745, x = var_1741)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_2609 = const()[name = tensor("op_2609"), val = tensor([6, 5, 256])]; - tensor attn_output_7 = reshape(shape = var_2609, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1749 = const()[name = tensor("op_1749"), val = tensor([6, 5, 256])]; + tensor attn_output_7 = reshape(shape = var_1749, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; - tensor input_335 = add(x = query_1, y = sa2_out_1)[name = tensor("input_335")]; - tensor input_337_axes_0 = const()[name = tensor("input_337_axes_0"), val = tensor([-1])]; - tensor input_337 = layer_norm(axes = input_337_axes_0, beta = decoder_norm21_0_bias, epsilon = var_2366, gamma = decoder_norm21_0_weight, x = input_335)[name = tensor("input_337")]; - tensor input_339 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_337)[name = tensor("linear_45")]; - tensor input_341 = relu(x = input_339)[name = tensor("input_341")]; - tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_341)[name = tensor("linear_46")]; - tensor input_343 = add(x = input_337, y = ffn_out_1)[name = tensor("input_343")]; + tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; + tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1505, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; + tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; + tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; + tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_2366, gamma = decoder_norm22_0_weight, x = input_343)[name = tensor("xt_3")]; - tensor var_2629 = const()[name = tensor("op_2629"), val = tensor([1, 5, 6, 256])]; - tensor x_63 = reshape(shape = var_2629, x = xt_3)[name = tensor("x_63")]; - tensor var_2631_perm_0 = const()[name = tensor("op_2631_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2635 = const()[name = tensor("op_2635"), val = tensor([6, 5, 256])]; - tensor var_2631 = transpose(perm = var_2631_perm_0, x = x_63)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_2635, x = var_2631)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1505, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1769 = const()[name = tensor("op_1769"), val = tensor([1, 5, 6, 256])]; + tensor x_31 = reshape(shape = var_1769, x = xt_3)[name = tensor("x_31")]; + tensor var_1771_perm_0 = const()[name = tensor("op_1771_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1775 = const()[name = tensor("op_1775"), val = tensor([6, 5, 256])]; + tensor var_1771 = transpose(perm = var_1771_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1775, x = var_1771)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 6, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -2315,120 +1700,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_2643 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_2644 = const()[name = tensor("op_2644"), val = tensor([6, 5, 4, 64])]; - tensor var_2645 = reshape(shape = var_2644, x = var_2643)[name = tensor("op_2645")]; + tensor var_1783 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1784 = const()[name = tensor("op_1784"), val = tensor([6, 5, 4, 64])]; + tensor var_1785 = reshape(shape = var_1784, x = var_1783)[name = tensor("op_1785")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2649 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_2650 = const()[name = tensor("op_2650"), val = tensor(0x1p-3)]; - tensor var_2651 = mul(x = var_2649, y = var_2650)[name = tensor("op_2651")]; - tensor var_2652 = const()[name = tensor("op_2652"), val = tensor([6, 5, 4, 64])]; - tensor var_2653 = reshape(shape = var_2652, x = var_2651)[name = tensor("op_2653")]; + tensor var_1789 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1790 = const()[name = tensor("op_1790"), val = tensor(0x1p-3)]; + tensor var_1791 = mul(x = var_1789, y = var_1790)[name = tensor("op_1791")]; + tensor var_1792 = const()[name = tensor("op_1792"), val = tensor([6, 5, 4, 64])]; + tensor var_1793 = reshape(shape = var_1792, x = var_1791)[name = tensor("op_1793")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2657 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_2658 = const()[name = tensor("op_2658"), val = tensor([6, 5, 4, 64])]; - tensor var_2659 = reshape(shape = var_2658, x = var_2657)[name = tensor("op_2659")]; + tensor var_1797 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1798 = const()[name = tensor("op_1798"), val = tensor([6, 5, 4, 64])]; + tensor var_1799 = reshape(shape = var_1798, x = var_1797)[name = tensor("op_1799")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_347 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; + tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; - tensor const_48 = const()[name = tensor("const_48"), val = tensor(0x1.fffffep+127)]; - tensor clip_43 = clip(alpha = var_2371, beta = const_48, x = s_t)[name = tensor("clip_43")]; + tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; + tensor clip_43 = clip(alpha = var_1510, beta = const_32, x = s_t)[name = tensor("clip_43")]; tensor sqrt_s_t = sqrt(x = clip_43)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_2653)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_2645)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1793)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1785)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_2674 = const()[name = tensor("op_2674"), val = tensor([5, 1])]; - tensor var_2675 = reshape(shape = var_2674, x = sqrt_s_t)[name = tensor("op_2675")]; - tensor M = real_div(x = causal_with_valid_1, y = var_2675)[name = tensor("M")]; - tensor var_2677 = mul(x = qk, y = M)[name = tensor("op_2677")]; + tensor var_1814 = const()[name = tensor("op_1814"), val = tensor([5, 1])]; + tensor var_1815 = reshape(shape = var_1814, x = sqrt_s_t)[name = tensor("op_1815")]; + tensor M = real_div(x = causal_with_valid_1, y = var_1815)[name = tensor("M")]; + tensor var_1817 = mul(x = qk, y = M)[name = tensor("op_1817")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_2659)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_2677, y = v_17)[name = tensor("inner")]; - tensor var_2679_transpose_x_0 = const()[name = tensor("op_2679_transpose_x_0"), val = tensor(false)]; - tensor var_2679_transpose_y_0 = const()[name = tensor("op_2679_transpose_y_0"), val = tensor(false)]; - tensor var_2679 = matmul(transpose_x = var_2679_transpose_x_0, transpose_y = var_2679_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_2679")]; - tensor var_2680 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_2680")]; - tensor var_2681 = const()[name = tensor("op_2681"), val = tensor([1, 1, 5, 1])]; - tensor var_2682 = reshape(shape = var_2681, x = var_2680)[name = tensor("op_2682")]; - tensor cross = mul(x = var_2679, y = var_2682)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1799)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1817, y = v_17)[name = tensor("inner")]; + tensor var_1819_transpose_x_0 = const()[name = tensor("op_1819_transpose_x_0"), val = tensor(false)]; + tensor var_1819_transpose_y_0 = const()[name = tensor("op_1819_transpose_y_0"), val = tensor(false)]; + tensor var_1819 = matmul(transpose_x = var_1819_transpose_x_0, transpose_y = var_1819_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1819")]; + tensor var_1820 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1820")]; + tensor var_1821 = const()[name = tensor("op_1821"), val = tensor([1, 1, 5, 1])]; + tensor var_1822 = reshape(shape = var_1821, x = var_1820)[name = tensor("op_1822")]; + tensor cross = mul(x = var_1819, y = var_1822)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_2505)[name = tensor("v_masked")]; - tensor var_2688 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_2688")]; - tensor var_2690_transpose_x_1 = const()[name = tensor("op_2690_transpose_x_1"), val = tensor(true)]; - tensor var_2690_transpose_y_1 = const()[name = tensor("op_2690_transpose_y_1"), val = tensor(false)]; - tensor var_2690 = matmul(transpose_x = var_2690_transpose_x_1, transpose_y = var_2690_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_2690")]; - tensor new_kv_unnorm = add(x = var_2688, y = var_2690)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_2513)[name = tensor("new_scale")]; - tensor const_49 = const()[name = tensor("const_49"), val = tensor(0x1.fffffep+127)]; - tensor clip_44 = clip(alpha = var_2371, beta = const_49, x = new_scale)[name = tensor("clip_44")]; + tensor v_masked = mul(x = v_17, y = var_1645)[name = tensor("v_masked")]; + tensor var_1828 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1828")]; + tensor var_1830_transpose_x_1 = const()[name = tensor("op_1830_transpose_x_1"), val = tensor(true)]; + tensor var_1830_transpose_y_1 = const()[name = tensor("op_1830_transpose_y_1"), val = tensor(false)]; + tensor var_1830 = matmul(transpose_x = var_1830_transpose_x_1, transpose_y = var_1830_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1830")]; + tensor new_kv_unnorm = add(x = var_1828, y = var_1830)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1653)[name = tensor("new_scale")]; + tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; + tensor clip_44 = clip(alpha = var_1510, beta = const_33, x = new_scale)[name = tensor("clip_44")]; tensor sqrt_new_scale = sqrt(x = clip_44)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_2699_perm_0 = const()[name = tensor("op_2699_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1839_perm_0 = const()[name = tensor("op_1839_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_2699 = transpose(perm = var_2699_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_2368, x = var_2699)[name = tensor("out_33")]; - tensor var_2703 = const()[name = tensor("op_2703"), val = tensor([6, 5, 256])]; - tensor out = reshape(shape = var_2703, x = out_33)[name = tensor("out")]; - tensor var_2705 = silu(x = input_347)[name = tensor("op_2705")]; - tensor input_349 = mul(x = var_2705, y = out)[name = tensor("input_349")]; - tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_349)[name = tensor("linear_51")]; - tensor input_351 = add(x = x, y = ret_out)[name = tensor("input_351")]; + tensor var_1839 = transpose(perm = var_1839_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1507, x = var_1839)[name = tensor("out_33")]; + tensor var_1843 = const()[name = tensor("op_1843"), val = tensor([6, 5, 256])]; + tensor out = reshape(shape = var_1843, x = out_33)[name = tensor("out")]; + tensor var_1845 = silu(x = input_187)[name = tensor("op_1845")]; + tensor input_189 = mul(x = var_1845, y = out)[name = tensor("input_189")]; + tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; + tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_2366, gamma = decoder_norm11_1_weight, x = input_351)[name = tensor("xt_5")]; - tensor var_2715 = const()[name = tensor("op_2715"), val = tensor([1, 6, 5, 256])]; - tensor var_2716 = reshape(shape = var_2715, x = xt_5)[name = tensor("op_2716")]; - tensor var_2717_perm_0 = const()[name = tensor("op_2717_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2720 = const()[name = tensor("op_2720"), val = tensor([5, 6, 256])]; - tensor var_2717 = transpose(perm = var_2717_perm_0, x = var_2716)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_2720, x = var_2717)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1505, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1855 = const()[name = tensor("op_1855"), val = tensor([1, 6, 5, 256])]; + tensor var_1856 = reshape(shape = var_1855, x = xt_5)[name = tensor("op_1856")]; + tensor var_1857_perm_0 = const()[name = tensor("op_1857_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1860 = const()[name = tensor("op_1860"), val = tensor([5, 6, 256])]; + tensor var_1857 = transpose(perm = var_1857_perm_0, x = var_1856)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1860, x = var_1857)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_2743 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_21 = const()[name = tensor("concat_21"), val = tensor([6, 5, 3, 256])]; - tensor var_2745 = reshape(shape = concat_21, x = var_2743)[name = tensor("op_2745")]; - tensor var_2746_axes_0 = const()[name = tensor("op_2746_axes_0"), val = tensor([0])]; - tensor var_2746 = expand_dims(axes = var_2746_axes_0, x = var_2745)[name = tensor("op_2746")]; - tensor var_2747_perm_0 = const()[name = tensor("op_2747_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2748_axes_0 = const()[name = tensor("op_2748_axes_0"), val = tensor([-2])]; - tensor var_2747 = transpose(perm = var_2747_perm_0, x = var_2746)[name = tensor("transpose_8")]; - tensor var_2748 = squeeze(axes = var_2748_axes_0, x = var_2747)[name = tensor("op_2748")]; + tensor var_1883 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_22 = const()[name = tensor("concat_22"), val = tensor([6, 5, 3, 256])]; + tensor var_1885 = reshape(shape = concat_22, x = var_1883)[name = tensor("op_1885")]; + tensor var_1886_axes_0 = const()[name = tensor("op_1886_axes_0"), val = tensor([0])]; + tensor var_1886 = expand_dims(axes = var_1886_axes_0, x = var_1885)[name = tensor("op_1886")]; + tensor var_1887_perm_0 = const()[name = tensor("op_1887_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1888_axes_0 = const()[name = tensor("op_1888_axes_0"), val = tensor([-2])]; + tensor var_1887 = transpose(perm = var_1887_perm_0, x = var_1886)[name = tensor("transpose_8")]; + tensor var_1888 = squeeze(axes = var_1888_axes_0, x = var_1887)[name = tensor("op_1888")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 6, 5, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_2748)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1888)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 6, 5, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_2748)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1888)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 6, 5, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_2748)[name = tensor("v_19")]; - tensor var_2756 = const()[name = tensor("op_2756"), val = tensor([6, 20, 64])]; - tensor var_2757 = reshape(shape = var_2756, x = q_19)[name = tensor("op_2757")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1888)[name = tensor("v_19")]; + tensor var_1896 = const()[name = tensor("op_1896"), val = tensor([6, 20, 64])]; + tensor var_1897 = reshape(shape = var_1896, x = q_19)[name = tensor("op_1897")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2763 = const()[name = tensor("op_2763"), val = tensor([6, 20, 64])]; - tensor var_2764 = reshape(shape = var_2763, x = k_19)[name = tensor("op_2764")]; + tensor var_1903 = const()[name = tensor("op_1903"), val = tensor([6, 20, 64])]; + tensor var_1904 = reshape(shape = var_1903, x = k_19)[name = tensor("op_1904")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2770 = const()[name = tensor("op_2770"), val = tensor([6, 20, 64])]; - tensor var_2771 = reshape(shape = var_2770, x = v_19)[name = tensor("op_2771")]; + tensor var_1910 = const()[name = tensor("op_1910"), val = tensor([6, 20, 64])]; + tensor var_1911 = reshape(shape = var_1910, x = v_19)[name = tensor("op_1911")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2774 = const()[name = tensor("op_2774"), val = tensor([5, 4, 6, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_2757)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_2774, x = q_21)[name = tensor("q")]; - tensor var_2776 = const()[name = tensor("op_2776"), val = tensor([5, 4, 6, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_2764)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_2776, x = k_21)[name = tensor("k")]; - tensor var_2778 = const()[name = tensor("op_2778"), val = tensor([5, 4, 6, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_2771)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_2778, x = v_21)[name = tensor("v")]; + tensor var_1914 = const()[name = tensor("op_1914"), val = tensor([5, 4, 6, 64])]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1897)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1914, x = q_21)[name = tensor("q")]; + tensor var_1916 = const()[name = tensor("op_1916"), val = tensor([5, 4, 6, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1904)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1916, x = k_21)[name = tensor("k")]; + tensor var_1918 = const()[name = tensor("op_1918"), val = tensor([5, 4, 6, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1911)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1918, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -2439,50 +1824,50 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_2781 = const()[name = tensor("op_2781"), val = tensor([2, 0, 1, 3])]; - tensor var_2786 = const()[name = tensor("op_2786"), val = tensor([30, 256])]; - tensor var_2782 = transpose(perm = var_2781, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_2786, x = var_2782)[name = tensor("attn_output_11")]; + tensor var_1921 = const()[name = tensor("op_1921"), val = tensor([2, 0, 1, 3])]; + tensor var_1926 = const()[name = tensor("op_1926"), val = tensor([30, 256])]; + tensor var_1922 = transpose(perm = var_1921, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1926, x = var_1922)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_2790 = const()[name = tensor("op_2790"), val = tensor([6, 5, 256])]; - tensor attn_output = reshape(shape = var_2790, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1930 = const()[name = tensor("op_1930"), val = tensor([6, 5, 256])]; + tensor attn_output = reshape(shape = var_1930, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; - tensor input_353 = add(x = query_5, y = sa2_out)[name = tensor("input_353")]; - tensor input_355_axes_0 = const()[name = tensor("input_355_axes_0"), val = tensor([-1])]; - tensor input_355 = layer_norm(axes = input_355_axes_0, beta = decoder_norm21_1_bias, epsilon = var_2366, gamma = decoder_norm21_1_weight, x = input_353)[name = tensor("input_355")]; - tensor input_357 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_355)[name = tensor("linear_54")]; - tensor input_359 = relu(x = input_357)[name = tensor("input_359")]; - tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_359)[name = tensor("linear_55")]; - tensor input_361 = add(x = input_355, y = ffn_out)[name = tensor("input_361")]; + tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; + tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1505, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; + tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; + tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; + tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_2366, gamma = decoder_norm22_1_weight, x = input_361)[name = tensor("xt")]; - tensor var_2810 = const()[name = tensor("op_2810"), val = tensor([1, 5, 6, 256])]; - tensor input = reshape(shape = var_2810, x = xt)[name = tensor("input")]; - tensor var_2812 = const()[name = tensor("op_2812"), val = tensor([-1])]; - tensor var_2813 = reduce_l2_norm(axes = var_2812, keep_dims = var_2369, x = input)[name = tensor("op_2813")]; - tensor const_58 = const()[name = tensor("const_58"), val = tensor(0x1.fffffep+127)]; - tensor clip_45 = clip(alpha = var_2360, beta = const_58, x = var_2813)[name = tensor("clip_45")]; - tensor var_2815 = real_div(x = input, y = clip_45)[name = tensor("op_2815")]; - tensor concat_25 = const()[name = tensor("concat_25"), val = tensor([5, 1, 256])]; - tensor reshape_0 = reshape(shape = concat_25, x = emb)[name = tensor("reshape_0")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1505, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1950 = const()[name = tensor("op_1950"), val = tensor([1, 5, 6, 256])]; + tensor input = reshape(shape = var_1950, x = xt)[name = tensor("input")]; + tensor var_1952 = const()[name = tensor("op_1952"), val = tensor([-1])]; + tensor var_1953 = reduce_l2_norm(axes = var_1952, keep_dims = var_1508, x = input)[name = tensor("op_1953")]; + tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; + tensor clip_45 = clip(alpha = var_1500, beta = const_42, x = var_1953)[name = tensor("clip_45")]; + tensor var_1955 = real_div(x = input, y = clip_45)[name = tensor("op_1955")]; + tensor concat_26 = const()[name = tensor("concat_26"), val = tensor([5, 1, 256])]; + tensor reshape_0 = reshape(shape = concat_26, x = emb)[name = tensor("reshape_0")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_26 = const()[name = tensor("concat_26"), val = tensor([5, 256, 6])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_2815)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_26, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_27 = const()[name = tensor("concat_27"), val = tensor([5, 256, 6])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1955)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_27, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = reshape_0, y = reshape_1)[name = tensor("matmul_2")]; - tensor concat_29 = const()[name = tensor("concat_29"), val = tensor([1, 5, 6])]; - tensor reshape_2 = reshape(shape = concat_29, x = matmul_2)[name = tensor("reshape_2")]; + tensor concat_30 = const()[name = tensor("concat_30"), val = tensor([1, 5, 6])]; + tensor reshape_2 = reshape(shape = concat_30, x = matmul_2)[name = tensor("reshape_2")]; tensor output_begin_0 = const()[name = tensor("output_begin_0"), val = tensor([0, 0, 1])]; tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 5, 5])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = reshape_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_2819")]; - tensor var_2821_axis_0 = const()[name = tensor("op_2821_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_2821_axis_0, values = (var_2517, nkv))[name = tensor("op_2821")]; - tensor var_2823_axis_0 = const()[name = tensor("op_2823_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_2823_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_2823")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1959")]; + tensor var_1961_axis_0 = const()[name = tensor("op_1961_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1961_axis_0, values = (var_1657, nkv))[name = tensor("op_1961")]; + tensor var_1963_axis_0 = const()[name = tensor("op_1963_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1963_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1963")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/AMI/ls_eend_ami_500ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/AMI/ls_eend_ami_500ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index a627ea0ff58f3ec3b508ae6f2d580a6261e58da0..af00933380589361a4bdc8710719ffd4e5fc5185 100644 --- a/optimized/AMI/ls_eend_ami_500ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/AMI/ls_eend_ami_500ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e2e81742dd8f8499a8d95b2926a750c99194e7233932596fd731ec86580a5adc -size 352353 +oid sha256:d51549a4f7b83e9f8cd566c1120746d77540795b6c9e8314dd5ee8d7ff9caf3e +size 255663 diff --git a/optimized/AMI/ls_eend_ami_500ms.mlpackage/Manifest.json b/optimized/AMI/ls_eend_ami_500ms.mlpackage/Manifest.json index 8dd225db34b52216e012a95d6ad1697d751f9b5e..8288909c9733ce737343458546717402fbab1b19 100644 --- a/optimized/AMI/ls_eend_ami_500ms.mlpackage/Manifest.json +++ b/optimized/AMI/ls_eend_ami_500ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "09C2BE97-6BB6-440E-961C-654D958D5874": { + "433B06B8-49E3-4838-BE30-CA31FDC93D2E": { "author": "com.apple.CoreML", "description": "CoreML Model Weights", "name": "weights", "path": "com.apple.CoreML/weights" }, - "8F43348E-9354-477B-921F-333C3EBCC222": { + "64A27ABF-4FD6-47B7-BEAB-C90CF7440F10": { "author": "com.apple.CoreML", "description": "CoreML Model Specification", "name": "model.mlmodel", "path": "com.apple.CoreML/model.mlmodel" } }, - "rootModelIdentifier": "8F43348E-9354-477B-921F-333C3EBCC222" + "rootModelIdentifier": "64A27ABF-4FD6-47B7-BEAB-C90CF7440F10" } diff --git a/optimized/CALLHOME/.DS_Store b/optimized/CALLHOME/.DS_Store index 594211e6ba52e7295da0eef7c07e5026a1111c53..dea355eb140d2b9cc6f822d5217310111c1201ad 100644 Binary files a/optimized/CALLHOME/.DS_Store and b/optimized/CALLHOME/.DS_Store differ diff --git a/optimized/CALLHOME/ls_eend_ch_100ms.mlmodelc/analytics/coremldata.bin b/optimized/CALLHOME/ls_eend_ch_100ms.mlmodelc/analytics/coremldata.bin index bfe3c10c252d359ae30bfad60919c17d2c2c3931..3c4a8c52917bfe3adf472c7d45b4b668af0b06f6 100644 --- a/optimized/CALLHOME/ls_eend_ch_100ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/CALLHOME/ls_eend_ch_100ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:754ef86accea44a257fbca5b14f6beb6d24dce6bd81e87253a6d3398492e5607 +oid sha256:39b9f768d22a20c2b16f3f429f4e8c066cee805ea805db3622598454840a46d9 size 243 diff --git a/optimized/CALLHOME/ls_eend_ch_100ms.mlmodelc/coremldata.bin b/optimized/CALLHOME/ls_eend_ch_100ms.mlmodelc/coremldata.bin index 37ef6402cf656f38182337c1cabdc077e306a1e9..2037ee0a00c8e9170786faf3eb4025269b57973d 100644 --- a/optimized/CALLHOME/ls_eend_ch_100ms.mlmodelc/coremldata.bin +++ b/optimized/CALLHOME/ls_eend_ch_100ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1eae2b912749fcd49e984ad436f5ee761f589d84452ec172aec523c1c96f38ce -size 1259 +oid sha256:b66e6e9177fe54da33043a4b3c00917a234a0fa1229d57644e55aca6aa387279 +size 1350 diff --git a/optimized/CALLHOME/ls_eend_ch_100ms.mlmodelc/metadata.json b/optimized/CALLHOME/ls_eend_ch_100ms.mlmodelc/metadata.json index 715abd481e7227c826ea70553271a81d4dcb0a03..9bbdf875e669851740be18a1c209e97af1d4f734 100644 --- a/optimized/CALLHOME/ls_eend_ch_100ms.mlmodelc/metadata.json +++ b/optimized/CALLHOME/ls_eend_ch_100ms.mlmodelc/metadata.json @@ -96,7 +96,7 @@ "Ios17.reshape" : 74, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 53, + "Ios17.transpose" : 57, "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 58, @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"ch\", \"model_label\": \"CALLHOME\", \"variant\": \"pipeline\", \"T\": 1, \"max_speakers\": 7, \"max_nspks\": 9, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"ch\", \"model_label\": \"CALLHOME\", \"variant\": \"pipeline\", \"chunk_size\": 1, \"step_duration_ms\": 100, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 7, \"max_nspks\": 9, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/CALLHOME/ls_eend_ch_100ms.mlmodelc/model.mil b/optimized/CALLHOME/ls_eend_ch_100ms.mlmodelc/model.mil index 9492b7b6f94a6decbe594c2d6033f36c61244e2c..e4e08882d3118bac7422ae4318e5b87ce7dac0aa 100644 --- a/optimized/CALLHOME/ls_eend_ch_100ms.mlmodelc/model.mil +++ b/optimized/CALLHOME/ls_eend_ch_100ms.mlmodelc/model.mil @@ -216,6 +216,7 @@ program(1.0) tensor var_21 = const()[name = tensor("op_21"), val = tensor(0x0p+0)]; tensor var_24 = const()[name = tensor("op_24"), val = tensor(0x1.0c6f7ap-20)]; tensor var_27 = const()[name = tensor("op_27"), val = tensor(2)]; + tensor var_30 = const()[name = tensor("op_30"), val = tensor(0)]; tensor var_33 = const()[name = tensor("op_33"), val = tensor(1)]; tensor var_35 = const()[name = tensor("op_35"), val = tensor(0x1.4f8b58p-17)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(true)]; @@ -262,8 +263,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_53")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_54")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_196 = const()[name = tensor("op_196"), val = tensor([1, 1])]; tensor var_197 = reshape(shape = var_196, x = sqrt_s_t_1)[name = tensor("op_197")]; @@ -271,7 +272,7 @@ program(1.0) tensor var_199 = mul(x = qk_1, y = M_1)[name = tensor("op_199")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_52")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_199, y = v_1)[name = tensor("inner_1")]; tensor var_201_transpose_x_0 = const()[name = tensor("op_201_transpose_x_0"), val = tensor(false)]; tensor var_201_transpose_y_0 = const()[name = tensor("op_201_transpose_y_0"), val = tensor(false)]; @@ -292,7 +293,7 @@ program(1.0) tensor var_214 = real_div(x = new_kv_unnorm_1, y = var_213)[name = tensor("op_214")]; tensor var_215_perm_0 = const()[name = tensor("op_215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_51")]; + tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_24, x = var_215)[name = tensor("out_3")]; tensor var_219 = const()[name = tensor("op_219"), val = tensor([1, 1, 256])]; tensor out_5 = reshape(shape = var_219, x = out_3)[name = tensor("out_5")]; @@ -336,10 +337,12 @@ program(1.0) tensor var_252 = sub(x = var_17, y = use_roll_1)[name = tensor("op_252")]; tensor var_253 = mul(x = filled_window_1, y = var_252)[name = tensor("op_253")]; tensor var_254 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_254")]; - tensor input_21 = add(x = var_253, y = var_254)[name = tensor("input_21")]; + tensor window_3 = add(x = var_253, y = var_254)[name = tensor("window_3")]; tensor var_256 = const()[name = tensor("op_256"), val = tensor(0x1p+0)]; tensor var_257 = add(x = filled_1, y = var_256)[name = tensor("op_257")]; tensor clip_1 = clip(alpha = var_21, beta = var_16, x = var_257)[name = tensor("clip_1")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_30, interleave = input_21_interleave_0, values = window_3)[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; @@ -348,13 +351,13 @@ program(1.0) tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_50")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_278_split_sizes_0 = const()[name = tensor("op_278_split_sizes_0"), val = tensor([256, 256])]; - tensor var_278_axis_0 = const()[name = tensor("op_278_axis_0"), val = tensor(1)]; - tensor var_278_0, tensor var_278_1 = split(axis = var_278_axis_0, split_sizes = var_278_split_sizes_0, x = inputs_3)[name = tensor("op_278")]; - tensor var_280 = sigmoid(x = var_278_1)[name = tensor("op_280")]; - tensor inputs_5 = mul(x = var_278_0, y = var_280)[name = tensor("inputs_5")]; + tensor var_280_split_sizes_0 = const()[name = tensor("op_280_split_sizes_0"), val = tensor([256, 256])]; + tensor var_280_axis_0 = const()[name = tensor("op_280_axis_0"), val = tensor(1)]; + tensor var_280_0, tensor var_280_1 = split(axis = var_280_axis_0, split_sizes = var_280_split_sizes_0, x = inputs_3)[name = tensor("op_280")]; + tensor var_282 = sigmoid(x = var_280_1)[name = tensor("op_282")]; + tensor inputs_5 = mul(x = var_280_0, y = var_282)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; @@ -374,22 +377,22 @@ program(1.0) tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_311_begin_0 = const()[name = tensor("op_311_begin_0"), val = tensor([0, -1, 0])]; - tensor var_311_end_0 = const()[name = tensor("op_311_end_0"), val = tensor([1, 16, 256])]; - tensor var_311_end_mask_0 = const()[name = tensor("op_311_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_49")]; - tensor var_311 = slice_by_index(begin = var_311_begin_0, end = var_311_end_0, end_mask = var_311_end_mask_0, x = conv_out_1)[name = tensor("op_311")]; - tensor var_314_interleave_0 = const()[name = tensor("op_314_interleave_0"), val = tensor(false)]; - tensor var_314 = concat(axis = var_33, interleave = var_314_interleave_0, values = var_311)[name = tensor("op_314")]; - tensor input_31 = add(x = x_3, y = var_314)[name = tensor("input_31")]; + tensor var_313_begin_0 = const()[name = tensor("op_313_begin_0"), val = tensor([0, -1, 0])]; + tensor var_313_end_0 = const()[name = tensor("op_313_end_0"), val = tensor([1, 16, 256])]; + tensor var_313_end_mask_0 = const()[name = tensor("op_313_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_313 = slice_by_index(begin = var_313_begin_0, end = var_313_end_0, end_mask = var_313_end_mask_0, x = conv_out_1)[name = tensor("op_313")]; + tensor var_315_perm_0 = const()[name = tensor("op_315_perm_0"), val = tensor([1, 0, 2])]; + tensor var_315 = transpose(perm = var_315_perm_0, x = var_313)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_315)[name = tensor("input_31")]; tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; - tensor var_337 = const()[name = tensor("op_337"), val = tensor(0x1p-1)]; - tensor var_338 = mul(x = input_39, y = var_337)[name = tensor("op_338")]; - tensor input_41 = add(x = var_338, y = input_31)[name = tensor("input_41")]; + tensor var_338 = const()[name = tensor("op_338"), val = tensor(0x1p-1)]; + tensor var_339 = mul(x = input_39, y = var_338)[name = tensor("op_339")]; + tensor input_41 = add(x = var_339, y = input_31)[name = tensor("input_41")]; tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; @@ -397,9 +400,9 @@ program(1.0) tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; - tensor var_367 = const()[name = tensor("op_367"), val = tensor(0x1p-1)]; - tensor var_368 = mul(x = input_51, y = var_367)[name = tensor("op_368")]; - tensor input_53 = add(x = var_368, y = input_43)[name = tensor("input_53")]; + tensor var_368 = const()[name = tensor("op_368"), val = tensor(0x1p-1)]; + tensor var_369 = mul(x = input_51, y = var_368)[name = tensor("op_369")]; + tensor input_53 = add(x = var_369, y = input_43)[name = tensor("input_53")]; tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; @@ -412,19 +415,19 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_382 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; - tensor var_383 = const()[name = tensor("op_383"), val = tensor([1, 1, 4, 64])]; - tensor var_384 = reshape(shape = var_383, x = var_382)[name = tensor("op_384")]; + tensor var_383 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_384 = const()[name = tensor("op_384"), val = tensor([1, 1, 4, 64])]; + tensor var_385 = reshape(shape = var_384, x = var_383)[name = tensor("op_385")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_388 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; - tensor var_389 = const()[name = tensor("op_389"), val = tensor(0x1p-3)]; - tensor var_390 = mul(x = var_388, y = var_389)[name = tensor("op_390")]; - tensor var_391 = const()[name = tensor("op_391"), val = tensor([1, 1, 4, 64])]; - tensor var_392 = reshape(shape = var_391, x = var_390)[name = tensor("op_392")]; + tensor var_389 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_390 = const()[name = tensor("op_390"), val = tensor(0x1p-3)]; + tensor var_391 = mul(x = var_389, y = var_390)[name = tensor("op_391")]; + tensor var_392 = const()[name = tensor("op_392"), val = tensor([1, 1, 4, 64])]; + tensor var_393 = reshape(shape = var_392, x = var_391)[name = tensor("op_393")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_396 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; - tensor var_397 = const()[name = tensor("op_397"), val = tensor([1, 1, 4, 64])]; - tensor var_398 = reshape(shape = var_397, x = var_396)[name = tensor("op_398")]; + tensor var_397 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_398 = const()[name = tensor("op_398"), val = tensor([1, 1, 4, 64])]; + tensor var_399 = reshape(shape = var_398, x = var_397)[name = tensor("op_399")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; @@ -432,84 +435,86 @@ program(1.0) tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_392)[name = tensor("transpose_47")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_384)[name = tensor("transpose_48")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_393)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_385)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_408 = const()[name = tensor("op_408"), val = tensor([1, 1])]; - tensor var_409 = reshape(shape = var_408, x = sqrt_s_t_3)[name = tensor("op_409")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_409)[name = tensor("M_3")]; - tensor var_411 = mul(x = qk_3, y = M_3)[name = tensor("op_411")]; + tensor var_409 = const()[name = tensor("op_409"), val = tensor([1, 1])]; + tensor var_410 = reshape(shape = var_409, x = sqrt_s_t_3)[name = tensor("op_410")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_410)[name = tensor("M_3")]; + tensor var_412 = mul(x = qk_3, y = M_3)[name = tensor("op_412")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_398)[name = tensor("transpose_46")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_411, y = v_3)[name = tensor("inner_3")]; - tensor var_413_transpose_x_0 = const()[name = tensor("op_413_transpose_x_0"), val = tensor(false)]; - tensor var_413_transpose_y_0 = const()[name = tensor("op_413_transpose_y_0"), val = tensor(false)]; - tensor var_413 = matmul(transpose_x = var_413_transpose_x_0, transpose_y = var_413_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_413")]; - tensor var_414 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_414")]; - tensor var_415 = const()[name = tensor("op_415"), val = tensor([1, 1, 1, 1])]; - tensor var_416 = reshape(shape = var_415, x = var_414)[name = tensor("op_416")]; - tensor cross_3 = mul(x = var_413, y = var_416)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_399)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_412, y = v_3)[name = tensor("inner_3")]; + tensor var_414_transpose_x_0 = const()[name = tensor("op_414_transpose_x_0"), val = tensor(false)]; + tensor var_414_transpose_y_0 = const()[name = tensor("op_414_transpose_y_0"), val = tensor(false)]; + tensor var_414 = matmul(transpose_x = var_414_transpose_x_0, transpose_y = var_414_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_414")]; + tensor var_415 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_415")]; + tensor var_416 = const()[name = tensor("op_416"), val = tensor([1, 1, 1, 1])]; + tensor var_417 = reshape(shape = var_416, x = var_415)[name = tensor("op_417")]; + tensor cross_3 = mul(x = var_414, y = var_417)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_419 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_419")]; - tensor var_421_transpose_x_1 = const()[name = tensor("op_421_transpose_x_1"), val = tensor(true)]; - tensor var_421_transpose_y_1 = const()[name = tensor("op_421_transpose_y_1"), val = tensor(false)]; - tensor var_421 = matmul(transpose_x = var_421_transpose_x_1, transpose_y = var_421_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_421")]; - tensor new_kv_unnorm_3 = add(x = var_419, y = var_421)[name = tensor("new_kv_unnorm_3")]; - tensor var_423 = const()[name = tensor("op_423"), val = tensor(0x1p+0)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_423)[name = tensor("new_scale_3")]; - tensor var_425 = sqrt(x = new_scale_3)[name = tensor("op_425")]; - tensor var_426 = real_div(x = new_kv_unnorm_3, y = var_425)[name = tensor("op_426")]; - tensor var_427_perm_0 = const()[name = tensor("op_427_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_420 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_420")]; + tensor var_422_transpose_x_1 = const()[name = tensor("op_422_transpose_x_1"), val = tensor(true)]; + tensor var_422_transpose_y_1 = const()[name = tensor("op_422_transpose_y_1"), val = tensor(false)]; + tensor var_422 = matmul(transpose_x = var_422_transpose_x_1, transpose_y = var_422_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_422")]; + tensor new_kv_unnorm_3 = add(x = var_420, y = var_422)[name = tensor("new_kv_unnorm_3")]; + tensor var_424 = const()[name = tensor("op_424"), val = tensor(0x1p+0)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_424)[name = tensor("new_scale_3")]; + tensor var_426 = sqrt(x = new_scale_3)[name = tensor("op_426")]; + tensor var_427 = real_div(x = new_kv_unnorm_3, y = var_426)[name = tensor("op_427")]; + tensor var_428_perm_0 = const()[name = tensor("op_428_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_427 = transpose(perm = var_427_perm_0, x = out_7)[name = tensor("transpose_45")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_427)[name = tensor("out_9")]; - tensor var_431 = const()[name = tensor("op_431"), val = tensor([1, 1, 256])]; - tensor out_11 = reshape(shape = var_431, x = out_9)[name = tensor("out_11")]; - tensor var_433 = silu(x = input_57)[name = tensor("op_433")]; - tensor input_59 = mul(x = var_433, y = out_11)[name = tensor("input_59")]; + tensor var_428 = transpose(perm = var_428_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_428)[name = tensor("out_9")]; + tensor var_432 = const()[name = tensor("op_432"), val = tensor([1, 1, 256])]; + tensor out_11 = reshape(shape = var_432, x = out_9)[name = tensor("out_11")]; + tensor var_434 = silu(x = input_57)[name = tensor("op_434")]; + tensor input_59 = mul(x = var_434, y = out_11)[name = tensor("input_59")]; tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; tensor filled_3_begin_0 = const()[name = tensor("filled_3_begin_0"), val = tensor([1, 0])]; tensor filled_3_end_0 = const()[name = tensor("filled_3_end_0"), val = tensor([2, 1])]; tensor filled_3_end_mask_0 = const()[name = tensor("filled_3_end_mask_0"), val = tensor([false, true])]; tensor filled_3_squeeze_mask_0 = const()[name = tensor("filled_3_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_3 = slice_by_index(begin = filled_3_begin_0, end = filled_3_end_0, end_mask = filled_3_end_mask_0, squeeze_mask = filled_3_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_3")]; - tensor var_445_begin_0 = const()[name = tensor("op_445_begin_0"), val = tensor([0, 1, 0])]; - tensor var_445_end_0 = const()[name = tensor("op_445_end_0"), val = tensor([1, 16, 256])]; - tensor var_445_end_mask_0 = const()[name = tensor("op_445_end_mask_0"), val = tensor([true, true, true])]; - tensor var_445 = slice_by_index(begin = var_445_begin_0, end = var_445_end_0, end_mask = var_445_end_mask_0, x = window_3)[name = tensor("op_445")]; + tensor var_446_begin_0 = const()[name = tensor("op_446_begin_0"), val = tensor([0, 1, 0])]; + tensor var_446_end_0 = const()[name = tensor("op_446_end_0"), val = tensor([1, 16, 256])]; + tensor var_446_end_mask_0 = const()[name = tensor("op_446_end_mask_0"), val = tensor([true, true, true])]; + tensor var_446 = slice_by_index(begin = var_446_begin_0, end = var_446_end_0, end_mask = var_446_end_mask_0, x = window_5)[name = tensor("op_446")]; tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_445, x_9))[name = tensor("rolled_3")]; + tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_446, x_9))[name = tensor("rolled_3")]; tensor clip_2 = clip(alpha = var_21, beta = var_20, x = filled_3)[name = tensor("clip_2")]; - tensor var_450 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_450")]; - tensor var_451_dtype_0 = const()[name = tensor("op_451_dtype_0"), val = tensor("fp32")]; - tensor var_452 = const()[name = tensor("op_452"), val = tensor([1, 16, 1])]; - tensor var_451 = cast(dtype = var_451_dtype_0, x = var_450)[name = tensor("cast_125")]; - tensor fill_mask_3 = reshape(shape = var_452, x = var_451)[name = tensor("fill_mask_3")]; + tensor var_451 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_451")]; + tensor var_452_dtype_0 = const()[name = tensor("op_452_dtype_0"), val = tensor("fp32")]; + tensor var_453 = const()[name = tensor("op_453"), val = tensor([1, 16, 1])]; + tensor var_452 = cast(dtype = var_452_dtype_0, x = var_451)[name = tensor("cast_125")]; + tensor fill_mask_3 = reshape(shape = var_453, x = var_452)[name = tensor("fill_mask_3")]; tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = x_9)[name = tensor("x_expanded_3")]; - tensor var_456 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_456")]; - tensor var_457 = mul(x = window_3, y = var_456)[name = tensor("op_457")]; - tensor var_458 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_458")]; - tensor filled_window_3 = add(x = var_457, y = var_458)[name = tensor("filled_window_3")]; - tensor var_460 = greater_equal(x = filled_3, y = var_16)[name = tensor("op_460")]; - tensor var_461_dtype_0 = const()[name = tensor("op_461_dtype_0"), val = tensor("fp32")]; - tensor var_462 = const()[name = tensor("op_462"), val = tensor([1, 1, 1])]; - tensor var_461 = cast(dtype = var_461_dtype_0, x = var_460)[name = tensor("cast_124")]; - tensor use_roll_3 = reshape(shape = var_462, x = var_461)[name = tensor("use_roll_3")]; - tensor var_464 = sub(x = var_17, y = use_roll_3)[name = tensor("op_464")]; - tensor var_465 = mul(x = filled_window_3, y = var_464)[name = tensor("op_465")]; - tensor var_466 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_466")]; - tensor input_61 = add(x = var_465, y = var_466)[name = tensor("input_61")]; - tensor var_468 = const()[name = tensor("op_468"), val = tensor(0x1p+0)]; - tensor var_469 = add(x = filled_3, y = var_468)[name = tensor("op_469")]; - tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_469)[name = tensor("clip_3")]; + tensor var_457 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_457")]; + tensor var_458 = mul(x = window_5, y = var_457)[name = tensor("op_458")]; + tensor var_459 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_459")]; + tensor filled_window_3 = add(x = var_458, y = var_459)[name = tensor("filled_window_3")]; + tensor var_461 = greater_equal(x = filled_3, y = var_16)[name = tensor("op_461")]; + tensor var_462_dtype_0 = const()[name = tensor("op_462_dtype_0"), val = tensor("fp32")]; + tensor var_463 = const()[name = tensor("op_463"), val = tensor([1, 1, 1])]; + tensor var_462 = cast(dtype = var_462_dtype_0, x = var_461)[name = tensor("cast_124")]; + tensor use_roll_3 = reshape(shape = var_463, x = var_462)[name = tensor("use_roll_3")]; + tensor var_465 = sub(x = var_17, y = use_roll_3)[name = tensor("op_465")]; + tensor var_466 = mul(x = filled_window_3, y = var_465)[name = tensor("op_466")]; + tensor var_467 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_467")]; + tensor window_7 = add(x = var_466, y = var_467)[name = tensor("window_7")]; + tensor var_469 = const()[name = tensor("op_469"), val = tensor(0x1p+0)]; + tensor var_470 = add(x = filled_3, y = var_469)[name = tensor("op_470")]; + tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_470)[name = tensor("clip_3")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_30, interleave = input_61_interleave_0, values = window_7)[name = tensor("input_61")]; tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; @@ -518,13 +523,13 @@ program(1.0) tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; - tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_44")]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; - tensor var_490_split_sizes_0 = const()[name = tensor("op_490_split_sizes_0"), val = tensor([256, 256])]; - tensor var_490_axis_0 = const()[name = tensor("op_490_axis_0"), val = tensor(1)]; - tensor var_490_0, tensor var_490_1 = split(axis = var_490_axis_0, split_sizes = var_490_split_sizes_0, x = inputs_13)[name = tensor("op_490")]; - tensor var_492 = sigmoid(x = var_490_1)[name = tensor("op_492")]; - tensor inputs_15 = mul(x = var_490_0, y = var_492)[name = tensor("inputs_15")]; + tensor var_493_split_sizes_0 = const()[name = tensor("op_493_split_sizes_0"), val = tensor([256, 256])]; + tensor var_493_axis_0 = const()[name = tensor("op_493_axis_0"), val = tensor(1)]; + tensor var_493_0, tensor var_493_1 = split(axis = var_493_axis_0, split_sizes = var_493_split_sizes_0, x = inputs_13)[name = tensor("op_493")]; + tensor var_495 = sigmoid(x = var_493_1)[name = tensor("op_495")]; + tensor inputs_15 = mul(x = var_493_0, y = var_495)[name = tensor("inputs_15")]; tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; @@ -544,22 +549,22 @@ program(1.0) tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_523_begin_0 = const()[name = tensor("op_523_begin_0"), val = tensor([0, -1, 0])]; - tensor var_523_end_0 = const()[name = tensor("op_523_end_0"), val = tensor([1, 16, 256])]; - tensor var_523_end_mask_0 = const()[name = tensor("op_523_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_43")]; - tensor var_523 = slice_by_index(begin = var_523_begin_0, end = var_523_end_0, end_mask = var_523_end_mask_0, x = conv_out_3)[name = tensor("op_523")]; - tensor var_526_interleave_0 = const()[name = tensor("op_526_interleave_0"), val = tensor(false)]; - tensor var_526 = concat(axis = var_33, interleave = var_526_interleave_0, values = var_523)[name = tensor("op_526")]; - tensor input_71 = add(x = x_9, y = var_526)[name = tensor("input_71")]; + tensor var_526_begin_0 = const()[name = tensor("op_526_begin_0"), val = tensor([0, -1, 0])]; + tensor var_526_end_0 = const()[name = tensor("op_526_end_0"), val = tensor([1, 16, 256])]; + tensor var_526_end_mask_0 = const()[name = tensor("op_526_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_526 = slice_by_index(begin = var_526_begin_0, end = var_526_end_0, end_mask = var_526_end_mask_0, x = conv_out_3)[name = tensor("op_526")]; + tensor var_528_perm_0 = const()[name = tensor("op_528_perm_0"), val = tensor([1, 0, 2])]; + tensor var_528 = transpose(perm = var_528_perm_0, x = var_526)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_528)[name = tensor("input_71")]; tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; - tensor var_549 = const()[name = tensor("op_549"), val = tensor(0x1p-1)]; - tensor var_550 = mul(x = input_79, y = var_549)[name = tensor("op_550")]; - tensor input_81 = add(x = var_550, y = input_71)[name = tensor("input_81")]; + tensor var_551 = const()[name = tensor("op_551"), val = tensor(0x1p-1)]; + tensor var_552 = mul(x = input_79, y = var_551)[name = tensor("op_552")]; + tensor input_81 = add(x = var_552, y = input_71)[name = tensor("input_81")]; tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; @@ -567,9 +572,9 @@ program(1.0) tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; - tensor var_579 = const()[name = tensor("op_579"), val = tensor(0x1p-1)]; - tensor var_580 = mul(x = input_91, y = var_579)[name = tensor("op_580")]; - tensor input_93 = add(x = var_580, y = input_83)[name = tensor("input_93")]; + tensor var_581 = const()[name = tensor("op_581"), val = tensor(0x1p-1)]; + tensor var_582 = mul(x = input_91, y = var_581)[name = tensor("op_582")]; + tensor input_93 = add(x = var_582, y = input_83)[name = tensor("input_93")]; tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; @@ -582,19 +587,19 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_594 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; - tensor var_595 = const()[name = tensor("op_595"), val = tensor([1, 1, 4, 64])]; - tensor var_596 = reshape(shape = var_595, x = var_594)[name = tensor("op_596")]; + tensor var_596 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_597 = const()[name = tensor("op_597"), val = tensor([1, 1, 4, 64])]; + tensor var_598 = reshape(shape = var_597, x = var_596)[name = tensor("op_598")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_600 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; - tensor var_601 = const()[name = tensor("op_601"), val = tensor(0x1p-3)]; - tensor var_602 = mul(x = var_600, y = var_601)[name = tensor("op_602")]; - tensor var_603 = const()[name = tensor("op_603"), val = tensor([1, 1, 4, 64])]; - tensor var_604 = reshape(shape = var_603, x = var_602)[name = tensor("op_604")]; + tensor var_602 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_603 = const()[name = tensor("op_603"), val = tensor(0x1p-3)]; + tensor var_604 = mul(x = var_602, y = var_603)[name = tensor("op_604")]; + tensor var_605 = const()[name = tensor("op_605"), val = tensor([1, 1, 4, 64])]; + tensor var_606 = reshape(shape = var_605, x = var_604)[name = tensor("op_606")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_608 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; - tensor var_609 = const()[name = tensor("op_609"), val = tensor([1, 1, 4, 64])]; - tensor var_610 = reshape(shape = var_609, x = var_608)[name = tensor("op_610")]; + tensor var_610 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_611 = const()[name = tensor("op_611"), val = tensor([1, 1, 4, 64])]; + tensor var_612 = reshape(shape = var_611, x = var_610)[name = tensor("op_612")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; @@ -602,84 +607,86 @@ program(1.0) tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_604)[name = tensor("transpose_41")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_596)[name = tensor("transpose_42")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_606)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_598)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_620 = const()[name = tensor("op_620"), val = tensor([1, 1])]; - tensor var_621 = reshape(shape = var_620, x = sqrt_s_t_5)[name = tensor("op_621")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_621)[name = tensor("M_5")]; - tensor var_623 = mul(x = qk_5, y = M_5)[name = tensor("op_623")]; + tensor var_622 = const()[name = tensor("op_622"), val = tensor([1, 1])]; + tensor var_623 = reshape(shape = var_622, x = sqrt_s_t_5)[name = tensor("op_623")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_623)[name = tensor("M_5")]; + tensor var_625 = mul(x = qk_5, y = M_5)[name = tensor("op_625")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_610)[name = tensor("transpose_40")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_623, y = v_5)[name = tensor("inner_5")]; - tensor var_625_transpose_x_0 = const()[name = tensor("op_625_transpose_x_0"), val = tensor(false)]; - tensor var_625_transpose_y_0 = const()[name = tensor("op_625_transpose_y_0"), val = tensor(false)]; - tensor var_625 = matmul(transpose_x = var_625_transpose_x_0, transpose_y = var_625_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_625")]; - tensor var_626 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_626")]; - tensor var_627 = const()[name = tensor("op_627"), val = tensor([1, 1, 1, 1])]; - tensor var_628 = reshape(shape = var_627, x = var_626)[name = tensor("op_628")]; - tensor cross_5 = mul(x = var_625, y = var_628)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_612)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_625, y = v_5)[name = tensor("inner_5")]; + tensor var_627_transpose_x_0 = const()[name = tensor("op_627_transpose_x_0"), val = tensor(false)]; + tensor var_627_transpose_y_0 = const()[name = tensor("op_627_transpose_y_0"), val = tensor(false)]; + tensor var_627 = matmul(transpose_x = var_627_transpose_x_0, transpose_y = var_627_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_627")]; + tensor var_628 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_628")]; + tensor var_629 = const()[name = tensor("op_629"), val = tensor([1, 1, 1, 1])]; + tensor var_630 = reshape(shape = var_629, x = var_628)[name = tensor("op_630")]; + tensor cross_5 = mul(x = var_627, y = var_630)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_631 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_631")]; - tensor var_633_transpose_x_1 = const()[name = tensor("op_633_transpose_x_1"), val = tensor(true)]; - tensor var_633_transpose_y_1 = const()[name = tensor("op_633_transpose_y_1"), val = tensor(false)]; - tensor var_633 = matmul(transpose_x = var_633_transpose_x_1, transpose_y = var_633_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_633")]; - tensor new_kv_unnorm_5 = add(x = var_631, y = var_633)[name = tensor("new_kv_unnorm_5")]; - tensor var_635 = const()[name = tensor("op_635"), val = tensor(0x1p+0)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_635)[name = tensor("new_scale_5")]; - tensor var_637 = sqrt(x = new_scale_5)[name = tensor("op_637")]; - tensor var_638 = real_div(x = new_kv_unnorm_5, y = var_637)[name = tensor("op_638")]; - tensor var_639_perm_0 = const()[name = tensor("op_639_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_633 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_633")]; + tensor var_635_transpose_x_1 = const()[name = tensor("op_635_transpose_x_1"), val = tensor(true)]; + tensor var_635_transpose_y_1 = const()[name = tensor("op_635_transpose_y_1"), val = tensor(false)]; + tensor var_635 = matmul(transpose_x = var_635_transpose_x_1, transpose_y = var_635_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_635")]; + tensor new_kv_unnorm_5 = add(x = var_633, y = var_635)[name = tensor("new_kv_unnorm_5")]; + tensor var_637 = const()[name = tensor("op_637"), val = tensor(0x1p+0)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_637)[name = tensor("new_scale_5")]; + tensor var_639 = sqrt(x = new_scale_5)[name = tensor("op_639")]; + tensor var_640 = real_div(x = new_kv_unnorm_5, y = var_639)[name = tensor("op_640")]; + tensor var_641_perm_0 = const()[name = tensor("op_641_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_639 = transpose(perm = var_639_perm_0, x = out_13)[name = tensor("transpose_39")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_639)[name = tensor("out_15")]; - tensor var_643 = const()[name = tensor("op_643"), val = tensor([1, 1, 256])]; - tensor out_17 = reshape(shape = var_643, x = out_15)[name = tensor("out_17")]; - tensor var_645 = silu(x = input_97)[name = tensor("op_645")]; - tensor input_99 = mul(x = var_645, y = out_17)[name = tensor("input_99")]; + tensor var_641 = transpose(perm = var_641_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_641)[name = tensor("out_15")]; + tensor var_645 = const()[name = tensor("op_645"), val = tensor([1, 1, 256])]; + tensor out_17 = reshape(shape = var_645, x = out_15)[name = tensor("out_17")]; + tensor var_647 = silu(x = input_97)[name = tensor("op_647")]; + tensor input_99 = mul(x = var_647, y = out_17)[name = tensor("input_99")]; tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor window_9_begin_0 = const()[name = tensor("window_9_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_9_end_0 = const()[name = tensor("window_9_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_9_end_mask_0 = const()[name = tensor("window_9_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_9_squeeze_mask_0 = const()[name = tensor("window_9_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_9 = slice_by_index(begin = window_9_begin_0, end = window_9_end_0, end_mask = window_9_end_mask_0, squeeze_mask = window_9_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_9")]; tensor filled_5_begin_0 = const()[name = tensor("filled_5_begin_0"), val = tensor([2, 0])]; tensor filled_5_end_0 = const()[name = tensor("filled_5_end_0"), val = tensor([3, 1])]; tensor filled_5_end_mask_0 = const()[name = tensor("filled_5_end_mask_0"), val = tensor([false, true])]; tensor filled_5_squeeze_mask_0 = const()[name = tensor("filled_5_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_5 = slice_by_index(begin = filled_5_begin_0, end = filled_5_end_0, end_mask = filled_5_end_mask_0, squeeze_mask = filled_5_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_5")]; - tensor var_657_begin_0 = const()[name = tensor("op_657_begin_0"), val = tensor([0, 1, 0])]; - tensor var_657_end_0 = const()[name = tensor("op_657_end_0"), val = tensor([1, 16, 256])]; - tensor var_657_end_mask_0 = const()[name = tensor("op_657_end_mask_0"), val = tensor([true, true, true])]; - tensor var_657 = slice_by_index(begin = var_657_begin_0, end = var_657_end_0, end_mask = var_657_end_mask_0, x = window_5)[name = tensor("op_657")]; + tensor var_659_begin_0 = const()[name = tensor("op_659_begin_0"), val = tensor([0, 1, 0])]; + tensor var_659_end_0 = const()[name = tensor("op_659_end_0"), val = tensor([1, 16, 256])]; + tensor var_659_end_mask_0 = const()[name = tensor("op_659_end_mask_0"), val = tensor([true, true, true])]; + tensor var_659 = slice_by_index(begin = var_659_begin_0, end = var_659_end_0, end_mask = var_659_end_mask_0, x = window_9)[name = tensor("op_659")]; tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_657, x_15))[name = tensor("rolled_5")]; + tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_659, x_15))[name = tensor("rolled_5")]; tensor clip_4 = clip(alpha = var_21, beta = var_20, x = filled_5)[name = tensor("clip_4")]; - tensor var_662 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_662")]; - tensor var_663_dtype_0 = const()[name = tensor("op_663_dtype_0"), val = tensor("fp32")]; - tensor var_664 = const()[name = tensor("op_664"), val = tensor([1, 16, 1])]; - tensor var_663 = cast(dtype = var_663_dtype_0, x = var_662)[name = tensor("cast_123")]; - tensor fill_mask_5 = reshape(shape = var_664, x = var_663)[name = tensor("fill_mask_5")]; + tensor var_664 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_664")]; + tensor var_665_dtype_0 = const()[name = tensor("op_665_dtype_0"), val = tensor("fp32")]; + tensor var_666 = const()[name = tensor("op_666"), val = tensor([1, 16, 1])]; + tensor var_665 = cast(dtype = var_665_dtype_0, x = var_664)[name = tensor("cast_123")]; + tensor fill_mask_5 = reshape(shape = var_666, x = var_665)[name = tensor("fill_mask_5")]; tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = x_15)[name = tensor("x_expanded_5")]; - tensor var_668 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_668")]; - tensor var_669 = mul(x = window_5, y = var_668)[name = tensor("op_669")]; - tensor var_670 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_670")]; - tensor filled_window_5 = add(x = var_669, y = var_670)[name = tensor("filled_window_5")]; - tensor var_672 = greater_equal(x = filled_5, y = var_16)[name = tensor("op_672")]; - tensor var_673_dtype_0 = const()[name = tensor("op_673_dtype_0"), val = tensor("fp32")]; - tensor var_674 = const()[name = tensor("op_674"), val = tensor([1, 1, 1])]; - tensor var_673 = cast(dtype = var_673_dtype_0, x = var_672)[name = tensor("cast_122")]; - tensor use_roll_5 = reshape(shape = var_674, x = var_673)[name = tensor("use_roll_5")]; - tensor var_676 = sub(x = var_17, y = use_roll_5)[name = tensor("op_676")]; - tensor var_677 = mul(x = filled_window_5, y = var_676)[name = tensor("op_677")]; - tensor var_678 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_678")]; - tensor input_101 = add(x = var_677, y = var_678)[name = tensor("input_101")]; - tensor var_680 = const()[name = tensor("op_680"), val = tensor(0x1p+0)]; - tensor var_681 = add(x = filled_5, y = var_680)[name = tensor("op_681")]; - tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_681)[name = tensor("clip_5")]; + tensor var_670 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_670")]; + tensor var_671 = mul(x = window_9, y = var_670)[name = tensor("op_671")]; + tensor var_672 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_672")]; + tensor filled_window_5 = add(x = var_671, y = var_672)[name = tensor("filled_window_5")]; + tensor var_674 = greater_equal(x = filled_5, y = var_16)[name = tensor("op_674")]; + tensor var_675_dtype_0 = const()[name = tensor("op_675_dtype_0"), val = tensor("fp32")]; + tensor var_676 = const()[name = tensor("op_676"), val = tensor([1, 1, 1])]; + tensor var_675 = cast(dtype = var_675_dtype_0, x = var_674)[name = tensor("cast_122")]; + tensor use_roll_5 = reshape(shape = var_676, x = var_675)[name = tensor("use_roll_5")]; + tensor var_678 = sub(x = var_17, y = use_roll_5)[name = tensor("op_678")]; + tensor var_679 = mul(x = filled_window_5, y = var_678)[name = tensor("op_679")]; + tensor var_680 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_680")]; + tensor window_11 = add(x = var_679, y = var_680)[name = tensor("window_11")]; + tensor var_682 = const()[name = tensor("op_682"), val = tensor(0x1p+0)]; + tensor var_683 = add(x = filled_5, y = var_682)[name = tensor("op_683")]; + tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_683)[name = tensor("clip_5")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_30, interleave = input_101_interleave_0, values = window_11)[name = tensor("input_101")]; tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; @@ -688,13 +695,13 @@ program(1.0) tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; - tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_38")]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; - tensor var_702_split_sizes_0 = const()[name = tensor("op_702_split_sizes_0"), val = tensor([256, 256])]; - tensor var_702_axis_0 = const()[name = tensor("op_702_axis_0"), val = tensor(1)]; - tensor var_702_0, tensor var_702_1 = split(axis = var_702_axis_0, split_sizes = var_702_split_sizes_0, x = inputs_23)[name = tensor("op_702")]; - tensor var_704 = sigmoid(x = var_702_1)[name = tensor("op_704")]; - tensor inputs_25 = mul(x = var_702_0, y = var_704)[name = tensor("inputs_25")]; + tensor var_706_split_sizes_0 = const()[name = tensor("op_706_split_sizes_0"), val = tensor([256, 256])]; + tensor var_706_axis_0 = const()[name = tensor("op_706_axis_0"), val = tensor(1)]; + tensor var_706_0, tensor var_706_1 = split(axis = var_706_axis_0, split_sizes = var_706_split_sizes_0, x = inputs_23)[name = tensor("op_706")]; + tensor var_708 = sigmoid(x = var_706_1)[name = tensor("op_708")]; + tensor inputs_25 = mul(x = var_706_0, y = var_708)[name = tensor("inputs_25")]; tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; @@ -714,22 +721,22 @@ program(1.0) tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_735_begin_0 = const()[name = tensor("op_735_begin_0"), val = tensor([0, -1, 0])]; - tensor var_735_end_0 = const()[name = tensor("op_735_end_0"), val = tensor([1, 16, 256])]; - tensor var_735_end_mask_0 = const()[name = tensor("op_735_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_37")]; - tensor var_735 = slice_by_index(begin = var_735_begin_0, end = var_735_end_0, end_mask = var_735_end_mask_0, x = conv_out_5)[name = tensor("op_735")]; - tensor var_738_interleave_0 = const()[name = tensor("op_738_interleave_0"), val = tensor(false)]; - tensor var_738 = concat(axis = var_33, interleave = var_738_interleave_0, values = var_735)[name = tensor("op_738")]; - tensor input_111 = add(x = x_15, y = var_738)[name = tensor("input_111")]; + tensor var_739_begin_0 = const()[name = tensor("op_739_begin_0"), val = tensor([0, -1, 0])]; + tensor var_739_end_0 = const()[name = tensor("op_739_end_0"), val = tensor([1, 16, 256])]; + tensor var_739_end_mask_0 = const()[name = tensor("op_739_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_739 = slice_by_index(begin = var_739_begin_0, end = var_739_end_0, end_mask = var_739_end_mask_0, x = conv_out_5)[name = tensor("op_739")]; + tensor var_741_perm_0 = const()[name = tensor("op_741_perm_0"), val = tensor([1, 0, 2])]; + tensor var_741 = transpose(perm = var_741_perm_0, x = var_739)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_741)[name = tensor("input_111")]; tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; - tensor var_761 = const()[name = tensor("op_761"), val = tensor(0x1p-1)]; - tensor var_762 = mul(x = input_119, y = var_761)[name = tensor("op_762")]; - tensor input_121 = add(x = var_762, y = input_111)[name = tensor("input_121")]; + tensor var_764 = const()[name = tensor("op_764"), val = tensor(0x1p-1)]; + tensor var_765 = mul(x = input_119, y = var_764)[name = tensor("op_765")]; + tensor input_121 = add(x = var_765, y = input_111)[name = tensor("input_121")]; tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; @@ -737,9 +744,9 @@ program(1.0) tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; - tensor var_791 = const()[name = tensor("op_791"), val = tensor(0x1p-1)]; - tensor var_792 = mul(x = input_131, y = var_791)[name = tensor("op_792")]; - tensor input_133 = add(x = var_792, y = input_123)[name = tensor("input_133")]; + tensor var_794 = const()[name = tensor("op_794"), val = tensor(0x1p-1)]; + tensor var_795 = mul(x = input_131, y = var_794)[name = tensor("op_795")]; + tensor input_133 = add(x = var_795, y = input_123)[name = tensor("input_133")]; tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; @@ -752,19 +759,19 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_806 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; - tensor var_807 = const()[name = tensor("op_807"), val = tensor([1, 1, 4, 64])]; - tensor var_808 = reshape(shape = var_807, x = var_806)[name = tensor("op_808")]; + tensor var_809 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_810 = const()[name = tensor("op_810"), val = tensor([1, 1, 4, 64])]; + tensor var_811 = reshape(shape = var_810, x = var_809)[name = tensor("op_811")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_812 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; - tensor var_813 = const()[name = tensor("op_813"), val = tensor(0x1p-3)]; - tensor var_814 = mul(x = var_812, y = var_813)[name = tensor("op_814")]; - tensor var_815 = const()[name = tensor("op_815"), val = tensor([1, 1, 4, 64])]; - tensor var_816 = reshape(shape = var_815, x = var_814)[name = tensor("op_816")]; + tensor var_815 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_816 = const()[name = tensor("op_816"), val = tensor(0x1p-3)]; + tensor var_817 = mul(x = var_815, y = var_816)[name = tensor("op_817")]; + tensor var_818 = const()[name = tensor("op_818"), val = tensor([1, 1, 4, 64])]; + tensor var_819 = reshape(shape = var_818, x = var_817)[name = tensor("op_819")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_820 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; - tensor var_821 = const()[name = tensor("op_821"), val = tensor([1, 1, 4, 64])]; - tensor var_822 = reshape(shape = var_821, x = var_820)[name = tensor("op_822")]; + tensor var_823 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_824 = const()[name = tensor("op_824"), val = tensor([1, 1, 4, 64])]; + tensor var_825 = reshape(shape = var_824, x = var_823)[name = tensor("op_825")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; @@ -772,84 +779,86 @@ program(1.0) tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_816)[name = tensor("transpose_35")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_808)[name = tensor("transpose_36")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_819)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_811)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_832 = const()[name = tensor("op_832"), val = tensor([1, 1])]; - tensor var_833 = reshape(shape = var_832, x = sqrt_s_t_7)[name = tensor("op_833")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_833)[name = tensor("M_7")]; - tensor var_835 = mul(x = qk_7, y = M_7)[name = tensor("op_835")]; + tensor var_835 = const()[name = tensor("op_835"), val = tensor([1, 1])]; + tensor var_836 = reshape(shape = var_835, x = sqrt_s_t_7)[name = tensor("op_836")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_836)[name = tensor("M_7")]; + tensor var_838 = mul(x = qk_7, y = M_7)[name = tensor("op_838")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_822)[name = tensor("transpose_34")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_835, y = v_7)[name = tensor("inner_7")]; - tensor var_837_transpose_x_0 = const()[name = tensor("op_837_transpose_x_0"), val = tensor(false)]; - tensor var_837_transpose_y_0 = const()[name = tensor("op_837_transpose_y_0"), val = tensor(false)]; - tensor var_837 = matmul(transpose_x = var_837_transpose_x_0, transpose_y = var_837_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_837")]; - tensor var_838 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_838")]; - tensor var_839 = const()[name = tensor("op_839"), val = tensor([1, 1, 1, 1])]; - tensor var_840 = reshape(shape = var_839, x = var_838)[name = tensor("op_840")]; - tensor cross_7 = mul(x = var_837, y = var_840)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_825)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_838, y = v_7)[name = tensor("inner_7")]; + tensor var_840_transpose_x_0 = const()[name = tensor("op_840_transpose_x_0"), val = tensor(false)]; + tensor var_840_transpose_y_0 = const()[name = tensor("op_840_transpose_y_0"), val = tensor(false)]; + tensor var_840 = matmul(transpose_x = var_840_transpose_x_0, transpose_y = var_840_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_840")]; + tensor var_841 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_841")]; + tensor var_842 = const()[name = tensor("op_842"), val = tensor([1, 1, 1, 1])]; + tensor var_843 = reshape(shape = var_842, x = var_841)[name = tensor("op_843")]; + tensor cross_7 = mul(x = var_840, y = var_843)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_843 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_843")]; - tensor var_845_transpose_x_1 = const()[name = tensor("op_845_transpose_x_1"), val = tensor(true)]; - tensor var_845_transpose_y_1 = const()[name = tensor("op_845_transpose_y_1"), val = tensor(false)]; - tensor var_845 = matmul(transpose_x = var_845_transpose_x_1, transpose_y = var_845_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_845")]; - tensor new_kv_unnorm_7 = add(x = var_843, y = var_845)[name = tensor("new_kv_unnorm_7")]; - tensor var_847 = const()[name = tensor("op_847"), val = tensor(0x1p+0)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_847)[name = tensor("new_scale_7")]; - tensor var_849 = sqrt(x = new_scale_7)[name = tensor("op_849")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_849)[name = tensor("nkv_1")]; - tensor var_851_perm_0 = const()[name = tensor("op_851_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_846 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_846")]; + tensor var_848_transpose_x_1 = const()[name = tensor("op_848_transpose_x_1"), val = tensor(true)]; + tensor var_848_transpose_y_1 = const()[name = tensor("op_848_transpose_y_1"), val = tensor(false)]; + tensor var_848 = matmul(transpose_x = var_848_transpose_x_1, transpose_y = var_848_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_848")]; + tensor new_kv_unnorm_7 = add(x = var_846, y = var_848)[name = tensor("new_kv_unnorm_7")]; + tensor var_850 = const()[name = tensor("op_850"), val = tensor(0x1p+0)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_850)[name = tensor("new_scale_7")]; + tensor var_852 = sqrt(x = new_scale_7)[name = tensor("op_852")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_852)[name = tensor("nkv_1")]; + tensor var_854_perm_0 = const()[name = tensor("op_854_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_851 = transpose(perm = var_851_perm_0, x = out_19)[name = tensor("transpose_33")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_851)[name = tensor("out_21")]; - tensor var_855 = const()[name = tensor("op_855"), val = tensor([1, 1, 256])]; - tensor out_23 = reshape(shape = var_855, x = out_21)[name = tensor("out_23")]; - tensor var_857 = silu(x = input_137)[name = tensor("op_857")]; - tensor input_139 = mul(x = var_857, y = out_23)[name = tensor("input_139")]; + tensor var_854 = transpose(perm = var_854_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_854)[name = tensor("out_21")]; + tensor var_858 = const()[name = tensor("op_858"), val = tensor([1, 1, 256])]; + tensor out_23 = reshape(shape = var_858, x = out_21)[name = tensor("out_23")]; + tensor var_860 = silu(x = input_137)[name = tensor("op_860")]; + tensor input_139 = mul(x = var_860, y = out_23)[name = tensor("input_139")]; tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor window_13_begin_0 = const()[name = tensor("window_13_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_13_end_0 = const()[name = tensor("window_13_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_13_end_mask_0 = const()[name = tensor("window_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_13_squeeze_mask_0 = const()[name = tensor("window_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_13 = slice_by_index(begin = window_13_begin_0, end = window_13_end_0, end_mask = window_13_end_mask_0, squeeze_mask = window_13_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_13")]; tensor filled_7_begin_0 = const()[name = tensor("filled_7_begin_0"), val = tensor([3, 0])]; tensor filled_7_end_0 = const()[name = tensor("filled_7_end_0"), val = tensor([4, 1])]; tensor filled_7_end_mask_0 = const()[name = tensor("filled_7_end_mask_0"), val = tensor([false, true])]; tensor filled_7_squeeze_mask_0 = const()[name = tensor("filled_7_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_7 = slice_by_index(begin = filled_7_begin_0, end = filled_7_end_0, end_mask = filled_7_end_mask_0, squeeze_mask = filled_7_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_7")]; - tensor var_869_begin_0 = const()[name = tensor("op_869_begin_0"), val = tensor([0, 1, 0])]; - tensor var_869_end_0 = const()[name = tensor("op_869_end_0"), val = tensor([1, 16, 256])]; - tensor var_869_end_mask_0 = const()[name = tensor("op_869_end_mask_0"), val = tensor([true, true, true])]; - tensor var_869 = slice_by_index(begin = var_869_begin_0, end = var_869_end_0, end_mask = var_869_end_mask_0, x = window)[name = tensor("op_869")]; + tensor var_872_begin_0 = const()[name = tensor("op_872_begin_0"), val = tensor([0, 1, 0])]; + tensor var_872_end_0 = const()[name = tensor("op_872_end_0"), val = tensor([1, 16, 256])]; + tensor var_872_end_mask_0 = const()[name = tensor("op_872_end_mask_0"), val = tensor([true, true, true])]; + tensor var_872 = slice_by_index(begin = var_872_begin_0, end = var_872_end_0, end_mask = var_872_end_mask_0, x = window_13)[name = tensor("op_872")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_869, x_21))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_872, x_21))[name = tensor("rolled")]; tensor clip_6 = clip(alpha = var_21, beta = var_20, x = filled_7)[name = tensor("clip_6")]; - tensor var_874 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_874")]; - tensor var_875_dtype_0 = const()[name = tensor("op_875_dtype_0"), val = tensor("fp32")]; - tensor var_876 = const()[name = tensor("op_876"), val = tensor([1, 16, 1])]; - tensor var_875 = cast(dtype = var_875_dtype_0, x = var_874)[name = tensor("cast_121")]; - tensor fill_mask = reshape(shape = var_876, x = var_875)[name = tensor("fill_mask")]; + tensor var_877 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_877")]; + tensor var_878_dtype_0 = const()[name = tensor("op_878_dtype_0"), val = tensor("fp32")]; + tensor var_879 = const()[name = tensor("op_879"), val = tensor([1, 16, 1])]; + tensor var_878 = cast(dtype = var_878_dtype_0, x = var_877)[name = tensor("cast_121")]; + tensor fill_mask = reshape(shape = var_879, x = var_878)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; tensor x_expanded = tile(reps = x_expanded_reps_0, x = x_21)[name = tensor("x_expanded")]; - tensor var_880 = sub(x = var_17, y = fill_mask)[name = tensor("op_880")]; - tensor var_881 = mul(x = window, y = var_880)[name = tensor("op_881")]; - tensor var_882 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_882")]; - tensor filled_window = add(x = var_881, y = var_882)[name = tensor("filled_window")]; - tensor var_884 = greater_equal(x = filled_7, y = var_16)[name = tensor("op_884")]; - tensor var_885_dtype_0 = const()[name = tensor("op_885_dtype_0"), val = tensor("fp32")]; - tensor var_886 = const()[name = tensor("op_886"), val = tensor([1, 1, 1])]; - tensor var_885 = cast(dtype = var_885_dtype_0, x = var_884)[name = tensor("cast_120")]; - tensor use_roll = reshape(shape = var_886, x = var_885)[name = tensor("use_roll")]; - tensor var_888 = sub(x = var_17, y = use_roll)[name = tensor("op_888")]; - tensor var_889 = mul(x = filled_window, y = var_888)[name = tensor("op_889")]; - tensor var_890 = mul(x = rolled, y = use_roll)[name = tensor("op_890")]; - tensor input_141 = add(x = var_889, y = var_890)[name = tensor("input_141")]; - tensor var_892 = const()[name = tensor("op_892"), val = tensor(0x1p+0)]; - tensor var_893 = add(x = filled_7, y = var_892)[name = tensor("op_893")]; - tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_893)[name = tensor("clip_7")]; + tensor var_883 = sub(x = var_17, y = fill_mask)[name = tensor("op_883")]; + tensor var_884 = mul(x = window_13, y = var_883)[name = tensor("op_884")]; + tensor var_885 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_885")]; + tensor filled_window = add(x = var_884, y = var_885)[name = tensor("filled_window")]; + tensor var_887 = greater_equal(x = filled_7, y = var_16)[name = tensor("op_887")]; + tensor var_888_dtype_0 = const()[name = tensor("op_888_dtype_0"), val = tensor("fp32")]; + tensor var_889 = const()[name = tensor("op_889"), val = tensor([1, 1, 1])]; + tensor var_888 = cast(dtype = var_888_dtype_0, x = var_887)[name = tensor("cast_120")]; + tensor use_roll = reshape(shape = var_889, x = var_888)[name = tensor("use_roll")]; + tensor var_891 = sub(x = var_17, y = use_roll)[name = tensor("op_891")]; + tensor var_892 = mul(x = filled_window, y = var_891)[name = tensor("op_892")]; + tensor var_893 = mul(x = rolled, y = use_roll)[name = tensor("op_893")]; + tensor window = add(x = var_892, y = var_893)[name = tensor("window")]; + tensor var_895 = const()[name = tensor("op_895"), val = tensor(0x1p+0)]; + tensor var_896 = add(x = filled_7, y = var_895)[name = tensor("op_896")]; + tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_896)[name = tensor("clip_7")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_30, interleave = input_141_interleave_0, values = window)[name = tensor("input_141")]; tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; @@ -858,13 +867,13 @@ program(1.0) tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; - tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_32")]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; - tensor var_914_split_sizes_0 = const()[name = tensor("op_914_split_sizes_0"), val = tensor([256, 256])]; - tensor var_914_axis_0 = const()[name = tensor("op_914_axis_0"), val = tensor(1)]; - tensor var_914_0, tensor var_914_1 = split(axis = var_914_axis_0, split_sizes = var_914_split_sizes_0, x = inputs_33)[name = tensor("op_914")]; - tensor var_916 = sigmoid(x = var_914_1)[name = tensor("op_916")]; - tensor inputs_35 = mul(x = var_914_0, y = var_916)[name = tensor("inputs_35")]; + tensor var_919_split_sizes_0 = const()[name = tensor("op_919_split_sizes_0"), val = tensor([256, 256])]; + tensor var_919_axis_0 = const()[name = tensor("op_919_axis_0"), val = tensor(1)]; + tensor var_919_0, tensor var_919_1 = split(axis = var_919_axis_0, split_sizes = var_919_split_sizes_0, x = inputs_33)[name = tensor("op_919")]; + tensor var_921 = sigmoid(x = var_919_1)[name = tensor("op_921")]; + tensor inputs_35 = mul(x = var_919_0, y = var_921)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; @@ -884,22 +893,22 @@ program(1.0) tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_947_begin_0 = const()[name = tensor("op_947_begin_0"), val = tensor([0, -1, 0])]; - tensor var_947_end_0 = const()[name = tensor("op_947_end_0"), val = tensor([1, 16, 256])]; - tensor var_947_end_mask_0 = const()[name = tensor("op_947_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_31")]; - tensor var_947 = slice_by_index(begin = var_947_begin_0, end = var_947_end_0, end_mask = var_947_end_mask_0, x = conv_out_7)[name = tensor("op_947")]; - tensor var_950_interleave_0 = const()[name = tensor("op_950_interleave_0"), val = tensor(false)]; - tensor var_950 = concat(axis = var_33, interleave = var_950_interleave_0, values = var_947)[name = tensor("op_950")]; - tensor input_151 = add(x = x_21, y = var_950)[name = tensor("input_151")]; + tensor var_952_begin_0 = const()[name = tensor("op_952_begin_0"), val = tensor([0, -1, 0])]; + tensor var_952_end_0 = const()[name = tensor("op_952_end_0"), val = tensor([1, 16, 256])]; + tensor var_952_end_mask_0 = const()[name = tensor("op_952_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_952 = slice_by_index(begin = var_952_begin_0, end = var_952_end_0, end_mask = var_952_end_mask_0, x = conv_out_7)[name = tensor("op_952")]; + tensor var_954_perm_0 = const()[name = tensor("op_954_perm_0"), val = tensor([1, 0, 2])]; + tensor var_954 = transpose(perm = var_954_perm_0, x = var_952)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_954)[name = tensor("input_151")]; tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; - tensor var_973 = const()[name = tensor("op_973"), val = tensor(0x1p-1)]; - tensor var_974 = mul(x = input_159, y = var_973)[name = tensor("op_974")]; - tensor input_161 = add(x = var_974, y = input_151)[name = tensor("input_161")]; + tensor var_977 = const()[name = tensor("op_977"), val = tensor(0x1p-1)]; + tensor var_978 = mul(x = input_159, y = var_977)[name = tensor("op_978")]; + tensor input_161 = add(x = var_978, y = input_151)[name = tensor("input_161")]; tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; @@ -912,44 +921,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_992_begin_0 = const()[name = tensor("op_992_begin_0"), val = tensor([0, 0, 1])]; - tensor var_992_end_0 = const()[name = tensor("op_992_end_0"), val = tensor([1, 256, 19])]; - tensor var_992_end_mask_0 = const()[name = tensor("op_992_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_992_begin_0, end = var_992_end_0, end_mask = var_992_end_mask_0, x = cat)[name = tensor("op_992")]; + tensor var_996_begin_0 = const()[name = tensor("op_996_begin_0"), val = tensor([0, 0, 1])]; + tensor var_996_end_0 = const()[name = tensor("op_996_end_0"), val = tensor([1, 256, 19])]; + tensor var_996_end_mask_0 = const()[name = tensor("op_996_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_996_begin_0, end = var_996_end_0, end_mask = var_996_end_mask_0, x = cat)[name = tensor("op_996")]; tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; - tensor var_994 = const()[name = tensor("op_994"), val = tensor([-1])]; + tensor var_998 = const()[name = tensor("op_998"), val = tensor([-1])]; tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_995 = reduce_l2_norm(axes = var_994, keep_dims = var_36, x = input_163)[name = tensor("op_995")]; + tensor var_999 = reduce_l2_norm(axes = var_998, keep_dims = var_36, x = input_163)[name = tensor("op_999")]; tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; - tensor clip_8 = clip(alpha = var_12, beta = const_12, x = var_995)[name = tensor("clip_8")]; + tensor clip_8 = clip(alpha = var_12, beta = const_12, x = var_999)[name = tensor("clip_8")]; tensor emb = real_div(x = input_163, y = clip_8)[name = tensor("emb")]; - tensor var_999_axis_0 = const()[name = tensor("op_999_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_999_axis_0, values = (var_214, var_426, var_638, nkv_1))[name = tensor("op_999")]; - tensor var_1001_axis_0 = const()[name = tensor("op_1001_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_1001_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1001")]; tensor var_1003_axis_0 = const()[name = tensor("op_1003_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_1003_axis_0, values = (input_21, input_61, input_101, input_141))[name = tensor("op_1003")]; + tensor enc_kv_new = stack(axis = var_1003_axis_0, values = (var_214, var_427, var_640, nkv_1))[name = tensor("op_1003")]; tensor var_1005_axis_0 = const()[name = tensor("op_1005_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_1005_axis_0, values = (clip_1, clip_3, clip_5, clip_7))[name = tensor("op_1005")]; - tensor var_1015 = const()[name = tensor("op_1015"), val = tensor(0x1.5798eep-27)]; - tensor var_1021 = const()[name = tensor("op_1021"), val = tensor(0x1.4f8b58p-17)]; - tensor var_1023 = const()[name = tensor("op_1023"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_1024 = const()[name = tensor("op_1024"), val = tensor(true)]; - tensor var_1026 = const()[name = tensor("op_1026"), val = tensor(0x1p+0)]; - tensor var_1030 = const()[name = tensor("op_1030"), val = tensor(-1)]; - tensor var_1035 = const()[name = tensor("op_1035"), val = tensor(0)]; - tensor var_1092 = const()[name = tensor("op_1092"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395456)))]; - tensor var_1097_axes_0 = const()[name = tensor("op_1097_axes_0"), val = tensor([2])]; - tensor var_1097 = expand_dims(axes = var_1097_axes_0, x = emb)[name = tensor("op_1097")]; - tensor var_1098 = const()[name = tensor("op_1098"), val = tensor([1, 1, 9, 1])]; - tensor emb_exp = tile(reps = var_1098, x = var_1097)[name = tensor("emb_exp")]; + tensor enc_scale_new = stack(axis = var_1005_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1005")]; + tensor var_1007_axis_0 = const()[name = tensor("op_1007_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1007_axis_0, values = (window_3, window_7, window_11, window))[name = tensor("op_1007")]; + tensor var_1009_axis_0 = const()[name = tensor("op_1009_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1009_axis_0, values = (clip_1, clip_3, clip_5, clip_7))[name = tensor("op_1009")]; + tensor var_1019 = const()[name = tensor("op_1019"), val = tensor(0x1.5798eep-27)]; + tensor var_1024 = const()[name = tensor("op_1024"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1026 = const()[name = tensor("op_1026"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1027 = const()[name = tensor("op_1027"), val = tensor(true)]; + tensor var_1029 = const()[name = tensor("op_1029"), val = tensor(0x1p+0)]; + tensor var_1033 = const()[name = tensor("op_1033"), val = tensor(-1)]; + tensor var_1039 = const()[name = tensor("op_1039"), val = tensor(0)]; + tensor var_1096 = const()[name = tensor("op_1096"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395456)))]; + tensor var_1101_axes_0 = const()[name = tensor("op_1101_axes_0"), val = tensor([2])]; + tensor var_1101 = expand_dims(axes = var_1101_axes_0, x = emb)[name = tensor("op_1101")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 9, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1101)[name = tensor("emb_exp")]; tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; - tensor input_165 = concat(axis = var_1030, interleave = input_165_interleave_0, values = (emb_exp, var_1092))[name = tensor("input_165")]; + tensor input_165 = concat(axis = var_1033, interleave = input_165_interleave_0, values = (emb_exp, var_1096))[name = tensor("input_165")]; tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; - tensor var_1105_perm_0 = const()[name = tensor("op_1105_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1109 = const()[name = tensor("op_1109"), val = tensor([9, 1, 256])]; - tensor var_1105 = transpose(perm = var_1105_perm_0, x = x_27)[name = tensor("transpose_28")]; - tensor x_29 = reshape(shape = var_1109, x = var_1105)[name = tensor("x_29")]; + tensor var_1109_perm_0 = const()[name = tensor("op_1109_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1113 = const()[name = tensor("op_1113"), val = tensor([9, 1, 256])]; + tensor var_1109 = transpose(perm = var_1109_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1113, x = var_1109)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 9, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -960,131 +969,131 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_1117 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; - tensor var_1118 = const()[name = tensor("op_1118"), val = tensor([9, 1, 4, 64])]; - tensor var_1119 = reshape(shape = var_1118, x = var_1117)[name = tensor("op_1119")]; + tensor var_1121 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1122 = const()[name = tensor("op_1122"), val = tensor([9, 1, 4, 64])]; + tensor var_1123 = reshape(shape = var_1122, x = var_1121)[name = tensor("op_1123")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1123 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; - tensor var_1124 = const()[name = tensor("op_1124"), val = tensor(0x1p-3)]; - tensor var_1125 = mul(x = var_1123, y = var_1124)[name = tensor("op_1125")]; - tensor var_1126 = const()[name = tensor("op_1126"), val = tensor([9, 1, 4, 64])]; - tensor var_1127 = reshape(shape = var_1126, x = var_1125)[name = tensor("op_1127")]; + tensor var_1127 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1128 = const()[name = tensor("op_1128"), val = tensor(0x1p-3)]; + tensor var_1129 = mul(x = var_1127, y = var_1128)[name = tensor("op_1129")]; + tensor var_1130 = const()[name = tensor("op_1130"), val = tensor([9, 1, 4, 64])]; + tensor var_1131 = reshape(shape = var_1130, x = var_1129)[name = tensor("op_1131")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1131 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; - tensor var_1132 = const()[name = tensor("op_1132"), val = tensor([9, 1, 4, 64])]; - tensor var_1133 = reshape(shape = var_1132, x = var_1131)[name = tensor("op_1133")]; + tensor var_1135 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1136 = const()[name = tensor("op_1136"), val = tensor([9, 1, 4, 64])]; + tensor var_1137 = reshape(shape = var_1136, x = var_1135)[name = tensor("op_1137")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_1035, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1039, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; - tensor clip_9 = clip(alpha = var_1026, beta = const_20, x = s_t_9)[name = tensor("clip_9")]; + tensor clip_9 = clip(alpha = var_1029, beta = const_20, x = s_t_9)[name = tensor("clip_9")]; tensor sqrt_s_t_9 = sqrt(x = clip_9)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_1127)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_1119)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1131)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1123)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_1145 = const()[name = tensor("op_1145"), val = tensor([1, 1])]; - tensor var_1146 = reshape(shape = var_1145, x = valid_mask)[name = tensor("op_1146")]; - tensor var_1148 = const()[name = tensor("op_1148"), val = tensor([1, 1])]; - tensor var_1149 = reshape(shape = var_1148, x = sqrt_s_t_9)[name = tensor("op_1149")]; - tensor M_9 = real_div(x = var_1146, y = var_1149)[name = tensor("M_9")]; - tensor var_1151 = mul(x = qk_9, y = M_9)[name = tensor("op_1151")]; + tensor var_1149 = const()[name = tensor("op_1149"), val = tensor([1, 1])]; + tensor var_1150 = reshape(shape = var_1149, x = valid_mask)[name = tensor("op_1150")]; + tensor var_1152 = const()[name = tensor("op_1152"), val = tensor([1, 1])]; + tensor var_1153 = reshape(shape = var_1152, x = sqrt_s_t_9)[name = tensor("op_1153")]; + tensor M_9 = real_div(x = var_1150, y = var_1153)[name = tensor("M_9")]; + tensor var_1155 = mul(x = qk_9, y = M_9)[name = tensor("op_1155")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_1133)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1151, y = v_9)[name = tensor("inner_9")]; - tensor var_1153_transpose_x_0 = const()[name = tensor("op_1153_transpose_x_0"), val = tensor(false)]; - tensor var_1153_transpose_y_0 = const()[name = tensor("op_1153_transpose_y_0"), val = tensor(false)]; - tensor var_1153 = matmul(transpose_x = var_1153_transpose_x_0, transpose_y = var_1153_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1153")]; - tensor var_1154 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1154")]; - tensor var_1155 = const()[name = tensor("op_1155"), val = tensor([1, 1, 1, 1])]; - tensor var_1156 = reshape(shape = var_1155, x = var_1154)[name = tensor("op_1156")]; - tensor cross_9 = mul(x = var_1153, y = var_1156)[name = tensor("cross_9")]; - tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1137)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1155, y = v_9)[name = tensor("inner_9")]; + tensor var_1157_transpose_x_0 = const()[name = tensor("op_1157_transpose_x_0"), val = tensor(false)]; + tensor var_1157_transpose_y_0 = const()[name = tensor("op_1157_transpose_y_0"), val = tensor(false)]; + tensor var_1157 = matmul(transpose_x = var_1157_transpose_x_0, transpose_y = var_1157_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1157")]; + tensor var_1158 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1158")]; tensor var_1159 = const()[name = tensor("op_1159"), val = tensor([1, 1, 1, 1])]; - tensor var_1160 = reshape(shape = var_1159, x = valid_mask)[name = tensor("op_1160")]; - tensor v_masked_1 = mul(x = v_9, y = var_1160)[name = tensor("v_masked_1")]; - tensor var_1162 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1162")]; - tensor var_1164_transpose_x_1 = const()[name = tensor("op_1164_transpose_x_1"), val = tensor(true)]; - tensor var_1164_transpose_y_1 = const()[name = tensor("op_1164_transpose_y_1"), val = tensor(false)]; - tensor var_1164 = matmul(transpose_x = var_1164_transpose_x_1, transpose_y = var_1164_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1164")]; - tensor new_kv_unnorm_9 = add(x = var_1162, y = var_1164)[name = tensor("new_kv_unnorm_9")]; - tensor var_1166_keep_dims_0 = const()[name = tensor("op_1166_keep_dims_0"), val = tensor(false)]; - tensor var_1166 = reduce_sum(keep_dims = var_1166_keep_dims_0, x = valid_mask)[name = tensor("op_1166")]; - tensor var_1167 = const()[name = tensor("op_1167"), val = tensor([1])]; - tensor var_1168 = reshape(shape = var_1167, x = var_1166)[name = tensor("op_1168")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_1168)[name = tensor("new_scale_9")]; + tensor var_1160 = reshape(shape = var_1159, x = var_1158)[name = tensor("op_1160")]; + tensor cross_9 = mul(x = var_1157, y = var_1160)[name = tensor("cross_9")]; + tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; + tensor var_1163 = const()[name = tensor("op_1163"), val = tensor([1, 1, 1, 1])]; + tensor var_1164 = reshape(shape = var_1163, x = valid_mask)[name = tensor("op_1164")]; + tensor v_masked_1 = mul(x = v_9, y = var_1164)[name = tensor("v_masked_1")]; + tensor var_1166 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1166")]; + tensor var_1168_transpose_x_1 = const()[name = tensor("op_1168_transpose_x_1"), val = tensor(true)]; + tensor var_1168_transpose_y_1 = const()[name = tensor("op_1168_transpose_y_1"), val = tensor(false)]; + tensor var_1168 = matmul(transpose_x = var_1168_transpose_x_1, transpose_y = var_1168_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1168")]; + tensor new_kv_unnorm_9 = add(x = var_1166, y = var_1168)[name = tensor("new_kv_unnorm_9")]; + tensor var_1170_keep_dims_0 = const()[name = tensor("op_1170_keep_dims_0"), val = tensor(false)]; + tensor var_1170 = reduce_sum(keep_dims = var_1170_keep_dims_0, x = valid_mask)[name = tensor("op_1170")]; + tensor var_1171 = const()[name = tensor("op_1171"), val = tensor([1])]; + tensor var_1172 = reshape(shape = var_1171, x = var_1170)[name = tensor("op_1172")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1172)[name = tensor("new_scale_9")]; tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; - tensor clip_10 = clip(alpha = var_1026, beta = const_21, x = new_scale_9)[name = tensor("clip_10")]; + tensor clip_10 = clip(alpha = var_1029, beta = const_21, x = new_scale_9)[name = tensor("clip_10")]; tensor sqrt_new_scale_1 = sqrt(x = clip_10)[name = tensor("sqrt_new_scale_1")]; - tensor var_1172 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1172")]; - tensor var_1173_perm_0 = const()[name = tensor("op_1173_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1176 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1176")]; + tensor var_1177_perm_0 = const()[name = tensor("op_1177_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_1173 = transpose(perm = var_1173_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1023, x = var_1173)[name = tensor("out_27")]; - tensor var_1177 = const()[name = tensor("op_1177"), val = tensor([9, 1, 256])]; - tensor out_29 = reshape(shape = var_1177, x = out_27)[name = tensor("out_29")]; - tensor var_1179 = silu(x = input_169)[name = tensor("op_1179")]; - tensor input_171 = mul(x = var_1179, y = out_29)[name = tensor("input_171")]; + tensor var_1177 = transpose(perm = var_1177_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1026, x = var_1177)[name = tensor("out_27")]; + tensor var_1181 = const()[name = tensor("op_1181"), val = tensor([9, 1, 256])]; + tensor out_29 = reshape(shape = var_1181, x = out_27)[name = tensor("out_29")]; + tensor var_1183 = silu(x = input_169)[name = tensor("op_1183")]; + tensor input_171 = mul(x = var_1183, y = out_29)[name = tensor("input_171")]; tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1021, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; - tensor var_1189 = const()[name = tensor("op_1189"), val = tensor([1, 9, 1, 256])]; - tensor var_1190 = reshape(shape = var_1189, x = xt_1)[name = tensor("op_1190")]; - tensor var_1191_perm_0 = const()[name = tensor("op_1191_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1194 = const()[name = tensor("op_1194"), val = tensor([1, 9, 256])]; - tensor var_1191 = transpose(perm = var_1191_perm_0, x = var_1190)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_1194, x = var_1191)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1024, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1193 = const()[name = tensor("op_1193"), val = tensor([1, 9, 1, 256])]; + tensor var_1194 = reshape(shape = var_1193, x = xt_1)[name = tensor("op_1194")]; + tensor var_1195_perm_0 = const()[name = tensor("op_1195_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1198 = const()[name = tensor("op_1198"), val = tensor([1, 9, 256])]; + tensor var_1195 = transpose(perm = var_1195_perm_0, x = var_1194)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1198, x = var_1195)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_1217 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_4 = const()[name = tensor("concat_4"), val = tensor([9, 1, 3, 256])]; - tensor var_1219 = reshape(shape = concat_4, x = var_1217)[name = tensor("op_1219")]; - tensor var_1220_axes_0 = const()[name = tensor("op_1220_axes_0"), val = tensor([0])]; - tensor var_1220 = expand_dims(axes = var_1220_axes_0, x = var_1219)[name = tensor("op_1220")]; - tensor var_1221_perm_0 = const()[name = tensor("op_1221_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1222_axes_0 = const()[name = tensor("op_1222_axes_0"), val = tensor([-2])]; - tensor var_1221 = transpose(perm = var_1221_perm_0, x = var_1220)[name = tensor("transpose_21")]; - tensor var_1222 = squeeze(axes = var_1222_axes_0, x = var_1221)[name = tensor("op_1222")]; + tensor var_1221 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_5 = const()[name = tensor("concat_5"), val = tensor([9, 1, 3, 256])]; + tensor var_1223 = reshape(shape = concat_5, x = var_1221)[name = tensor("op_1223")]; + tensor var_1224_axes_0 = const()[name = tensor("op_1224_axes_0"), val = tensor([0])]; + tensor var_1224 = expand_dims(axes = var_1224_axes_0, x = var_1223)[name = tensor("op_1224")]; + tensor var_1225_perm_0 = const()[name = tensor("op_1225_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1226_axes_0 = const()[name = tensor("op_1226_axes_0"), val = tensor([-2])]; + tensor var_1225 = transpose(perm = var_1225_perm_0, x = var_1224)[name = tensor("transpose_21")]; + tensor var_1226 = squeeze(axes = var_1226_axes_0, x = var_1225)[name = tensor("op_1226")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 9, 1, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1222)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1226)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 9, 1, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1222)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1226)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 9, 1, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1222)[name = tensor("v_11")]; - tensor var_1230 = const()[name = tensor("op_1230"), val = tensor([9, 4, 64])]; - tensor var_1231 = reshape(shape = var_1230, x = q_11)[name = tensor("op_1231")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1226)[name = tensor("v_11")]; + tensor var_1234 = const()[name = tensor("op_1234"), val = tensor([9, 4, 64])]; + tensor var_1235 = reshape(shape = var_1234, x = q_11)[name = tensor("op_1235")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1237 = const()[name = tensor("op_1237"), val = tensor([9, 4, 64])]; - tensor var_1238 = reshape(shape = var_1237, x = k_11)[name = tensor("op_1238")]; + tensor var_1241 = const()[name = tensor("op_1241"), val = tensor([9, 4, 64])]; + tensor var_1242 = reshape(shape = var_1241, x = k_11)[name = tensor("op_1242")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1244 = const()[name = tensor("op_1244"), val = tensor([9, 4, 64])]; - tensor var_1245 = reshape(shape = var_1244, x = v_11)[name = tensor("op_1245")]; + tensor var_1248 = const()[name = tensor("op_1248"), val = tensor([9, 4, 64])]; + tensor var_1249 = reshape(shape = var_1248, x = v_11)[name = tensor("op_1249")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1248 = const()[name = tensor("op_1248"), val = tensor([1, 4, 9, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_1231)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_1248, x = q_13)[name = tensor("q_15")]; - tensor var_1250 = const()[name = tensor("op_1250"), val = tensor([1, 4, 9, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_1238)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_1250, x = k_13)[name = tensor("k_15")]; tensor var_1252 = const()[name = tensor("op_1252"), val = tensor([1, 4, 9, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_1245)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_1252, x = v_13)[name = tensor("v_15")]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1235)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1252, x = q_13)[name = tensor("q_15")]; + tensor var_1254 = const()[name = tensor("op_1254"), val = tensor([1, 4, 9, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1242)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1254, x = k_13)[name = tensor("k_15")]; + tensor var_1256 = const()[name = tensor("op_1256"), val = tensor([1, 4, 9, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1249)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1256, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -1095,30 +1104,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_1255 = const()[name = tensor("op_1255"), val = tensor([2, 0, 1, 3])]; - tensor var_1260 = const()[name = tensor("op_1260"), val = tensor([9, 256])]; - tensor var_1256 = transpose(perm = var_1255, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_1260, x = var_1256)[name = tensor("attn_output_3")]; + tensor var_1259 = const()[name = tensor("op_1259"), val = tensor([2, 0, 1, 3])]; + tensor var_1264 = const()[name = tensor("op_1264"), val = tensor([9, 256])]; + tensor var_1260 = transpose(perm = var_1259, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1264, x = var_1260)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_1264 = const()[name = tensor("op_1264"), val = tensor([9, 1, 256])]; - tensor attn_output_7 = reshape(shape = var_1264, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1268 = const()[name = tensor("op_1268"), val = tensor([9, 1, 256])]; + tensor attn_output_7 = reshape(shape = var_1268, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; - tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1021, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1024, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1021, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; - tensor var_1284 = const()[name = tensor("op_1284"), val = tensor([1, 1, 9, 256])]; - tensor x_31 = reshape(shape = var_1284, x = xt_3)[name = tensor("x_31")]; - tensor var_1286_perm_0 = const()[name = tensor("op_1286_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1290 = const()[name = tensor("op_1290"), val = tensor([9, 1, 256])]; - tensor var_1286 = transpose(perm = var_1286_perm_0, x = x_31)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_1290, x = var_1286)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1024, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1288 = const()[name = tensor("op_1288"), val = tensor([1, 1, 9, 256])]; + tensor x_31 = reshape(shape = var_1288, x = xt_3)[name = tensor("x_31")]; + tensor var_1290_perm_0 = const()[name = tensor("op_1290_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1294 = const()[name = tensor("op_1294"), val = tensor([9, 1, 256])]; + tensor var_1290 = transpose(perm = var_1290_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1294, x = var_1290)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 9, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1129,120 +1138,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_1298 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_1299 = const()[name = tensor("op_1299"), val = tensor([9, 1, 4, 64])]; - tensor var_1300 = reshape(shape = var_1299, x = var_1298)[name = tensor("op_1300")]; + tensor var_1302 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1303 = const()[name = tensor("op_1303"), val = tensor([9, 1, 4, 64])]; + tensor var_1304 = reshape(shape = var_1303, x = var_1302)[name = tensor("op_1304")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1304 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_1305 = const()[name = tensor("op_1305"), val = tensor(0x1p-3)]; - tensor var_1306 = mul(x = var_1304, y = var_1305)[name = tensor("op_1306")]; - tensor var_1307 = const()[name = tensor("op_1307"), val = tensor([9, 1, 4, 64])]; - tensor var_1308 = reshape(shape = var_1307, x = var_1306)[name = tensor("op_1308")]; + tensor var_1308 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1309 = const()[name = tensor("op_1309"), val = tensor(0x1p-3)]; + tensor var_1310 = mul(x = var_1308, y = var_1309)[name = tensor("op_1310")]; + tensor var_1311 = const()[name = tensor("op_1311"), val = tensor([9, 1, 4, 64])]; + tensor var_1312 = reshape(shape = var_1311, x = var_1310)[name = tensor("op_1312")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1312 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_1313 = const()[name = tensor("op_1313"), val = tensor([9, 1, 4, 64])]; - tensor var_1314 = reshape(shape = var_1313, x = var_1312)[name = tensor("op_1314")]; + tensor var_1316 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1317 = const()[name = tensor("op_1317"), val = tensor([9, 1, 4, 64])]; + tensor var_1318 = reshape(shape = var_1317, x = var_1316)[name = tensor("op_1318")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; - tensor clip_11 = clip(alpha = var_1026, beta = const_32, x = s_t)[name = tensor("clip_11")]; + tensor clip_11 = clip(alpha = var_1029, beta = const_32, x = s_t)[name = tensor("clip_11")]; tensor sqrt_s_t = sqrt(x = clip_11)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_1308)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_1300)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1312)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1304)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_1329 = const()[name = tensor("op_1329"), val = tensor([1, 1])]; - tensor var_1330 = reshape(shape = var_1329, x = sqrt_s_t)[name = tensor("op_1330")]; - tensor M = real_div(x = var_1146, y = var_1330)[name = tensor("M")]; - tensor var_1332 = mul(x = qk, y = M)[name = tensor("op_1332")]; + tensor var_1333 = const()[name = tensor("op_1333"), val = tensor([1, 1])]; + tensor var_1334 = reshape(shape = var_1333, x = sqrt_s_t)[name = tensor("op_1334")]; + tensor M = real_div(x = var_1150, y = var_1334)[name = tensor("M")]; + tensor var_1336 = mul(x = qk, y = M)[name = tensor("op_1336")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_1314)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1332, y = v_17)[name = tensor("inner")]; - tensor var_1334_transpose_x_0 = const()[name = tensor("op_1334_transpose_x_0"), val = tensor(false)]; - tensor var_1334_transpose_y_0 = const()[name = tensor("op_1334_transpose_y_0"), val = tensor(false)]; - tensor var_1334 = matmul(transpose_x = var_1334_transpose_x_0, transpose_y = var_1334_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1334")]; - tensor var_1335 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1335")]; - tensor var_1336 = const()[name = tensor("op_1336"), val = tensor([1, 1, 1, 1])]; - tensor var_1337 = reshape(shape = var_1336, x = var_1335)[name = tensor("op_1337")]; - tensor cross = mul(x = var_1334, y = var_1337)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1318)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1336, y = v_17)[name = tensor("inner")]; + tensor var_1338_transpose_x_0 = const()[name = tensor("op_1338_transpose_x_0"), val = tensor(false)]; + tensor var_1338_transpose_y_0 = const()[name = tensor("op_1338_transpose_y_0"), val = tensor(false)]; + tensor var_1338 = matmul(transpose_x = var_1338_transpose_x_0, transpose_y = var_1338_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1338")]; + tensor var_1339 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1339")]; + tensor var_1340 = const()[name = tensor("op_1340"), val = tensor([1, 1, 1, 1])]; + tensor var_1341 = reshape(shape = var_1340, x = var_1339)[name = tensor("op_1341")]; + tensor cross = mul(x = var_1338, y = var_1341)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_1160)[name = tensor("v_masked")]; - tensor var_1343 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1343")]; - tensor var_1345_transpose_x_1 = const()[name = tensor("op_1345_transpose_x_1"), val = tensor(true)]; - tensor var_1345_transpose_y_1 = const()[name = tensor("op_1345_transpose_y_1"), val = tensor(false)]; - tensor var_1345 = matmul(transpose_x = var_1345_transpose_x_1, transpose_y = var_1345_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1345")]; - tensor new_kv_unnorm = add(x = var_1343, y = var_1345)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_1168)[name = tensor("new_scale")]; + tensor v_masked = mul(x = v_17, y = var_1164)[name = tensor("v_masked")]; + tensor var_1347 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1347")]; + tensor var_1349_transpose_x_1 = const()[name = tensor("op_1349_transpose_x_1"), val = tensor(true)]; + tensor var_1349_transpose_y_1 = const()[name = tensor("op_1349_transpose_y_1"), val = tensor(false)]; + tensor var_1349 = matmul(transpose_x = var_1349_transpose_x_1, transpose_y = var_1349_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1349")]; + tensor new_kv_unnorm = add(x = var_1347, y = var_1349)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1172)[name = tensor("new_scale")]; tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; - tensor clip_12 = clip(alpha = var_1026, beta = const_33, x = new_scale)[name = tensor("clip_12")]; + tensor clip_12 = clip(alpha = var_1029, beta = const_33, x = new_scale)[name = tensor("clip_12")]; tensor sqrt_new_scale = sqrt(x = clip_12)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_1354_perm_0 = const()[name = tensor("op_1354_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1358_perm_0 = const()[name = tensor("op_1358_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_1354 = transpose(perm = var_1354_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1023, x = var_1354)[name = tensor("out_33")]; - tensor var_1358 = const()[name = tensor("op_1358"), val = tensor([9, 1, 256])]; - tensor out = reshape(shape = var_1358, x = out_33)[name = tensor("out")]; - tensor var_1360 = silu(x = input_187)[name = tensor("op_1360")]; - tensor input_189 = mul(x = var_1360, y = out)[name = tensor("input_189")]; + tensor var_1358 = transpose(perm = var_1358_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1026, x = var_1358)[name = tensor("out_33")]; + tensor var_1362 = const()[name = tensor("op_1362"), val = tensor([9, 1, 256])]; + tensor out = reshape(shape = var_1362, x = out_33)[name = tensor("out")]; + tensor var_1364 = silu(x = input_187)[name = tensor("op_1364")]; + tensor input_189 = mul(x = var_1364, y = out)[name = tensor("input_189")]; tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1021, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; - tensor var_1370 = const()[name = tensor("op_1370"), val = tensor([1, 9, 1, 256])]; - tensor var_1371 = reshape(shape = var_1370, x = xt_5)[name = tensor("op_1371")]; - tensor var_1372_perm_0 = const()[name = tensor("op_1372_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1375 = const()[name = tensor("op_1375"), val = tensor([1, 9, 256])]; - tensor var_1372 = transpose(perm = var_1372_perm_0, x = var_1371)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_1375, x = var_1372)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1024, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1374 = const()[name = tensor("op_1374"), val = tensor([1, 9, 1, 256])]; + tensor var_1375 = reshape(shape = var_1374, x = xt_5)[name = tensor("op_1375")]; + tensor var_1376_perm_0 = const()[name = tensor("op_1376_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1379 = const()[name = tensor("op_1379"), val = tensor([1, 9, 256])]; + tensor var_1376 = transpose(perm = var_1376_perm_0, x = var_1375)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1379, x = var_1376)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_1398 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_5 = const()[name = tensor("concat_5"), val = tensor([9, 1, 3, 256])]; - tensor var_1400 = reshape(shape = concat_5, x = var_1398)[name = tensor("op_1400")]; - tensor var_1401_axes_0 = const()[name = tensor("op_1401_axes_0"), val = tensor([0])]; - tensor var_1401 = expand_dims(axes = var_1401_axes_0, x = var_1400)[name = tensor("op_1401")]; - tensor var_1402_perm_0 = const()[name = tensor("op_1402_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1403_axes_0 = const()[name = tensor("op_1403_axes_0"), val = tensor([-2])]; - tensor var_1402 = transpose(perm = var_1402_perm_0, x = var_1401)[name = tensor("transpose_8")]; - tensor var_1403 = squeeze(axes = var_1403_axes_0, x = var_1402)[name = tensor("op_1403")]; + tensor var_1402 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_6 = const()[name = tensor("concat_6"), val = tensor([9, 1, 3, 256])]; + tensor var_1404 = reshape(shape = concat_6, x = var_1402)[name = tensor("op_1404")]; + tensor var_1405_axes_0 = const()[name = tensor("op_1405_axes_0"), val = tensor([0])]; + tensor var_1405 = expand_dims(axes = var_1405_axes_0, x = var_1404)[name = tensor("op_1405")]; + tensor var_1406_perm_0 = const()[name = tensor("op_1406_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1407_axes_0 = const()[name = tensor("op_1407_axes_0"), val = tensor([-2])]; + tensor var_1406 = transpose(perm = var_1406_perm_0, x = var_1405)[name = tensor("transpose_8")]; + tensor var_1407 = squeeze(axes = var_1407_axes_0, x = var_1406)[name = tensor("op_1407")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 9, 1, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1403)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1407)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 9, 1, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1403)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1407)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 9, 1, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1403)[name = tensor("v_19")]; - tensor var_1411 = const()[name = tensor("op_1411"), val = tensor([9, 4, 64])]; - tensor var_1412 = reshape(shape = var_1411, x = q_19)[name = tensor("op_1412")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1407)[name = tensor("v_19")]; + tensor var_1415 = const()[name = tensor("op_1415"), val = tensor([9, 4, 64])]; + tensor var_1416 = reshape(shape = var_1415, x = q_19)[name = tensor("op_1416")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1418 = const()[name = tensor("op_1418"), val = tensor([9, 4, 64])]; - tensor var_1419 = reshape(shape = var_1418, x = k_19)[name = tensor("op_1419")]; + tensor var_1422 = const()[name = tensor("op_1422"), val = tensor([9, 4, 64])]; + tensor var_1423 = reshape(shape = var_1422, x = k_19)[name = tensor("op_1423")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1425 = const()[name = tensor("op_1425"), val = tensor([9, 4, 64])]; - tensor var_1426 = reshape(shape = var_1425, x = v_19)[name = tensor("op_1426")]; + tensor var_1429 = const()[name = tensor("op_1429"), val = tensor([9, 4, 64])]; + tensor var_1430 = reshape(shape = var_1429, x = v_19)[name = tensor("op_1430")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1429 = const()[name = tensor("op_1429"), val = tensor([1, 4, 9, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_1412)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_1429, x = q_21)[name = tensor("q")]; - tensor var_1431 = const()[name = tensor("op_1431"), val = tensor([1, 4, 9, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_1419)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_1431, x = k_21)[name = tensor("k")]; tensor var_1433 = const()[name = tensor("op_1433"), val = tensor([1, 4, 9, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_1426)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_1433, x = v_21)[name = tensor("v")]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1416)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1433, x = q_21)[name = tensor("q")]; + tensor var_1435 = const()[name = tensor("op_1435"), val = tensor([1, 4, 9, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1423)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1435, x = k_21)[name = tensor("k")]; + tensor var_1437 = const()[name = tensor("op_1437"), val = tensor([1, 4, 9, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1430)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1437, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -1253,35 +1262,35 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_1436 = const()[name = tensor("op_1436"), val = tensor([2, 0, 1, 3])]; - tensor var_1441 = const()[name = tensor("op_1441"), val = tensor([9, 256])]; - tensor var_1437 = transpose(perm = var_1436, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_1441, x = var_1437)[name = tensor("attn_output_11")]; + tensor var_1440 = const()[name = tensor("op_1440"), val = tensor([2, 0, 1, 3])]; + tensor var_1445 = const()[name = tensor("op_1445"), val = tensor([9, 256])]; + tensor var_1441 = transpose(perm = var_1440, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1445, x = var_1441)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_1445 = const()[name = tensor("op_1445"), val = tensor([9, 1, 256])]; - tensor attn_output = reshape(shape = var_1445, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1449 = const()[name = tensor("op_1449"), val = tensor([9, 1, 256])]; + tensor attn_output = reshape(shape = var_1449, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; - tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1021, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1024, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1021, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; - tensor var_1465 = const()[name = tensor("op_1465"), val = tensor([1, 1, 9, 256])]; - tensor input = reshape(shape = var_1465, x = xt)[name = tensor("input")]; - tensor var_1467 = const()[name = tensor("op_1467"), val = tensor([-1])]; - tensor var_1468 = reduce_l2_norm(axes = var_1467, keep_dims = var_1024, x = input)[name = tensor("op_1468")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1024, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1469 = const()[name = tensor("op_1469"), val = tensor([1, 1, 9, 256])]; + tensor input = reshape(shape = var_1469, x = xt)[name = tensor("input")]; + tensor var_1471 = const()[name = tensor("op_1471"), val = tensor([-1])]; + tensor var_1472 = reduce_l2_norm(axes = var_1471, keep_dims = var_1027, x = input)[name = tensor("op_1472")]; tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; - tensor clip_13 = clip(alpha = var_1015, beta = const_42, x = var_1468)[name = tensor("clip_13")]; - tensor var_1470 = real_div(x = input, y = clip_13)[name = tensor("op_1470")]; + tensor clip_13 = clip(alpha = var_1019, beta = const_42, x = var_1472)[name = tensor("clip_13")]; + tensor var_1474 = real_div(x = input, y = clip_13)[name = tensor("op_1474")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_10 = const()[name = tensor("concat_10"), val = tensor([1, 256, 9])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1470)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_10, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_11 = const()[name = tensor("concat_11"), val = tensor([1, 256, 9])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1474)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_11, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = emb, y = reshape_1)[name = tensor("matmul_2")]; @@ -1289,10 +1298,10 @@ program(1.0) tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 1, 8])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = matmul_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_1474")]; - tensor var_1476_axis_0 = const()[name = tensor("op_1476_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_1476_axis_0, values = (var_1172, nkv))[name = tensor("op_1476")]; - tensor var_1478_axis_0 = const()[name = tensor("op_1478_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_1478_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1478")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1478")]; + tensor var_1480_axis_0 = const()[name = tensor("op_1480_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1480_axis_0, values = (var_1176, nkv))[name = tensor("op_1480")]; + tensor var_1482_axis_0 = const()[name = tensor("op_1482_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1482_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1482")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/CALLHOME/ls_eend_ch_100ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/CALLHOME/ls_eend_ch_100ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index e383de9d1a4c26bf9b2187007a1711218914c279..c5036fa5191c31c15387d52eed3b6e13abf947ae 100644 --- a/optimized/CALLHOME/ls_eend_ch_100ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/CALLHOME/ls_eend_ch_100ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bf30706fa3c43d451befc18aec080a73b582fed79f159a8d970a6731f53b0764 -size 184728 +oid sha256:772f41564ce75bb8d07137b6f8c2e6dfaed05a53093a99171e345634673f7c14 +size 185986 diff --git a/optimized/CALLHOME/ls_eend_ch_100ms.mlpackage/Manifest.json b/optimized/CALLHOME/ls_eend_ch_100ms.mlpackage/Manifest.json index c4d60959393775a1ccc1d381fe630639e8cc3291..94feddde05d64afa6ae4f6c18d52581573b4a31b 100644 --- a/optimized/CALLHOME/ls_eend_ch_100ms.mlpackage/Manifest.json +++ b/optimized/CALLHOME/ls_eend_ch_100ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "45347A98-ED26-4BF6-9585-9933823192BD": { + "5E72C85E-C893-442B-AAC9-008F6339D3AD": { "author": "com.apple.CoreML", "description": "CoreML Model Weights", "name": "weights", "path": "com.apple.CoreML/weights" }, - "ECF1C978-AF84-4914-9315-6B409AC8FA41": { + "6D3A8925-E8CA-41D6-A64E-8016A3CE384B": { "author": "com.apple.CoreML", "description": "CoreML Model Specification", "name": "model.mlmodel", "path": "com.apple.CoreML/model.mlmodel" } }, - "rootModelIdentifier": "ECF1C978-AF84-4914-9315-6B409AC8FA41" + "rootModelIdentifier": "6D3A8925-E8CA-41D6-A64E-8016A3CE384B" } diff --git a/optimized/CALLHOME/ls_eend_ch_200ms.mlmodelc/analytics/coremldata.bin b/optimized/CALLHOME/ls_eend_ch_200ms.mlmodelc/analytics/coremldata.bin index 2480bc3e8ab5641f659699168fd243fa960e2ed1..779cf7829fb3cdc18a6554572c90bff704f9555c 100644 --- a/optimized/CALLHOME/ls_eend_ch_200ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/CALLHOME/ls_eend_ch_200ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9421c7ca549a2b24d2cbb6f8ae82ba8ca75817d9405c774147ffbf75350eae70 +oid sha256:372addb8ae07073caa0c6885852a989aa2bdeedf024196c2e7725a4d62083d91 size 243 diff --git a/optimized/CALLHOME/ls_eend_ch_200ms.mlmodelc/coremldata.bin b/optimized/CALLHOME/ls_eend_ch_200ms.mlmodelc/coremldata.bin index def2f54e3b60b5cce55bd0a913aff2e2bcd0d798..de92ed85334f0ece749029b5d4eae99dcc33997c 100644 --- a/optimized/CALLHOME/ls_eend_ch_200ms.mlmodelc/coremldata.bin +++ b/optimized/CALLHOME/ls_eend_ch_200ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f07d2401fedeb80531c46dca4db8c3acf4ba6a72f2aeaf5eb50fceb2934ec231 -size 1259 +oid sha256:a3887a31e94a62264f697adf14cb6c924051d5ec2757898c6b6bf46da17716b5 +size 1350 diff --git a/optimized/CALLHOME/ls_eend_ch_200ms.mlmodelc/metadata.json b/optimized/CALLHOME/ls_eend_ch_200ms.mlmodelc/metadata.json index de4a9a01d275a145854180f97fbdf30d4b1e9bda..1408aa2416760f841f07ab71bd7f99ba8e64bb72 100644 --- a/optimized/CALLHOME/ls_eend_ch_200ms.mlmodelc/metadata.json +++ b/optimized/CALLHOME/ls_eend_ch_200ms.mlmodelc/metadata.json @@ -96,30 +96,30 @@ "Ios17.reshape" : 84, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 61, - "Split" : 8, + "Ios17.transpose" : 57, + "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 70, "Tile" : 9, - "Ios17.sliceByIndex" : 60, - "Ios16.sigmoid" : 9, + "Ios17.sliceByIndex" : 52, + "Ios16.sigmoid" : 5, "Ios16.reduceSum" : 1, "Ios17.squeeze" : 2, - "Ios17.layerNorm" : 37, - "Ios17.batchNorm" : 8, + "Ios17.layerNorm" : 33, + "Ios17.batchNorm" : 4, "Ios17.cast" : 16, "Ios17.sqrt" : 18, "Ios17.sub" : 16, - "Ios17.conv" : 25, + "Ios17.conv" : 13, "Ios17.realDiv" : 20, "Ios17.linear" : 56, "Ios17.concat" : 14, "Ios17.clip" : 22, "Ios17.greaterEqual" : 8, - "Ios16.silu" : 22, + "Ios16.silu" : 18, "Stack" : 6, "Ios16.cumsum" : 1, - "Ios17.mul" : 83 + "Ios17.mul" : 79 }, "computePrecision" : "Mixed (Float32, Int32)", "isUpdatable" : "0", @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"ch\", \"model_label\": \"CALLHOME\", \"variant\": \"pipeline\", \"T\": 2, \"max_speakers\": 7, \"max_nspks\": 9, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"ch\", \"model_label\": \"CALLHOME\", \"variant\": \"pipeline\", \"chunk_size\": 2, \"step_duration_ms\": 200, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 7, \"max_nspks\": 9, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/CALLHOME/ls_eend_ch_200ms.mlmodelc/model.mil b/optimized/CALLHOME/ls_eend_ch_200ms.mlmodelc/model.mil index 4611d0043932f8aa2a2e7b58ad604f8e2906d555..e99ffbf7453874c45d7313c1d0075e486decada4 100644 --- a/optimized/CALLHOME/ls_eend_ch_200ms.mlmodelc/model.mil +++ b/optimized/CALLHOME/ls_eend_ch_200ms.mlmodelc/model.mil @@ -217,6 +217,7 @@ program(1.0) tensor var_21 = const()[name = tensor("op_21"), val = tensor(0x0p+0)]; tensor var_24 = const()[name = tensor("op_24"), val = tensor(0x1.0c6f7ap-20)]; tensor var_27 = const()[name = tensor("op_27"), val = tensor(2)]; + tensor var_30 = const()[name = tensor("op_30"), val = tensor(0)]; tensor var_33 = const()[name = tensor("op_33"), val = tensor(1)]; tensor var_35 = const()[name = tensor("op_35"), val = tensor(0x1.4f8b58p-17)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(true)]; @@ -263,8 +264,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_61")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_62")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_196 = const()[name = tensor("op_196"), val = tensor([2, 1])]; tensor var_197 = reshape(shape = var_196, x = sqrt_s_t_1)[name = tensor("op_197")]; @@ -272,7 +273,7 @@ program(1.0) tensor var_199 = mul(x = qk_1, y = M_1)[name = tensor("op_199")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_60")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_199, y = v_1)[name = tensor("inner_1")]; tensor var_201_transpose_x_0 = const()[name = tensor("op_201_transpose_x_0"), val = tensor(false)]; tensor var_201_transpose_y_0 = const()[name = tensor("op_201_transpose_y_0"), val = tensor(false)]; @@ -293,7 +294,7 @@ program(1.0) tensor var_214 = real_div(x = new_kv_unnorm_1, y = var_213)[name = tensor("op_214")]; tensor var_215_perm_0 = const()[name = tensor("op_215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_59")]; + tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_24, x = var_215)[name = tensor("out_3")]; tensor var_219 = const()[name = tensor("op_219"), val = tensor([1, 2, 256])]; tensor out_5 = reshape(shape = var_219, x = out_3)[name = tensor("out_5")]; @@ -341,145 +342,108 @@ program(1.0) tensor var_252 = sub(x = var_17, y = use_roll_1)[name = tensor("op_252")]; tensor var_253 = mul(x = filled_window_1, y = var_252)[name = tensor("op_253")]; tensor var_254 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_254")]; - tensor input_21 = add(x = var_253, y = var_254)[name = tensor("input_21")]; + tensor window_3 = add(x = var_253, y = var_254)[name = tensor("window_3")]; tensor var_256 = const()[name = tensor("op_256"), val = tensor(0x1p+0)]; tensor var_257 = add(x = filled_1, y = var_256)[name = tensor("op_257")]; tensor clip_1 = clip(alpha = var_21, beta = var_16, x = var_257)[name = tensor("clip_1")]; + tensor var_260_begin_0 = const()[name = tensor("op_260_begin_0"), val = tensor([0, 1, 0])]; + tensor var_260_end_0 = const()[name = tensor("op_260_end_0"), val = tensor([1, 1, 256])]; + tensor var_260_end_mask_0 = const()[name = tensor("op_260_end_mask_0"), val = tensor([true, true, true])]; + tensor var_260 = slice_by_index(begin = var_260_begin_0, end = var_260_end_0, end_mask = var_260_end_mask_0, x = x_3)[name = tensor("op_260")]; + tensor var_263_begin_0 = const()[name = tensor("op_263_begin_0"), val = tensor([0, 1, 0])]; + tensor var_263_end_0 = const()[name = tensor("op_263_end_0"), val = tensor([1, 16, 256])]; + tensor var_263_end_mask_0 = const()[name = tensor("op_263_end_mask_0"), val = tensor([true, true, true])]; + tensor var_263 = slice_by_index(begin = var_263_begin_0, end = var_263_end_0, end_mask = var_263_end_mask_0, x = window_3)[name = tensor("op_263")]; + tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; + tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_263, var_260))[name = tensor("rolled_3")]; + tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; + tensor var_268 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_268")]; + tensor var_269_dtype_0 = const()[name = tensor("op_269_dtype_0"), val = tensor("fp32")]; + tensor var_270 = const()[name = tensor("op_270"), val = tensor([1, 16, 1])]; + tensor var_269 = cast(dtype = var_269_dtype_0, x = var_268)[name = tensor("cast_141")]; + tensor fill_mask_3 = reshape(shape = var_270, x = var_269)[name = tensor("fill_mask_3")]; + tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_260)[name = tensor("x_expanded_3")]; + tensor var_274 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_274")]; + tensor var_275 = mul(x = window_3, y = var_274)[name = tensor("op_275")]; + tensor var_276 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_276")]; + tensor filled_window_3 = add(x = var_275, y = var_276)[name = tensor("filled_window_3")]; + tensor var_278 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_278")]; + tensor var_279_dtype_0 = const()[name = tensor("op_279_dtype_0"), val = tensor("fp32")]; + tensor var_280 = const()[name = tensor("op_280"), val = tensor([1, 1, 1])]; + tensor var_279 = cast(dtype = var_279_dtype_0, x = var_278)[name = tensor("cast_140")]; + tensor use_roll_3 = reshape(shape = var_280, x = var_279)[name = tensor("use_roll_3")]; + tensor var_282 = sub(x = var_17, y = use_roll_3)[name = tensor("op_282")]; + tensor var_283 = mul(x = filled_window_3, y = var_282)[name = tensor("op_283")]; + tensor var_284 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_284")]; + tensor window_5 = add(x = var_283, y = var_284)[name = tensor("window_5")]; + tensor var_286 = const()[name = tensor("op_286"), val = tensor(0x1p+0)]; + tensor var_287 = add(x = clip_1, y = var_286)[name = tensor("op_287")]; + tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_287)[name = tensor("clip_3")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_30, interleave = input_21_interleave_0, values = (window_3, window_5))[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; - tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; + tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; tensor inputs_3_pad_type_0 = const()[name = tensor("inputs_3_pad_type_0"), val = tensor("valid")]; tensor inputs_3_strides_0 = const()[name = tensor("inputs_3_strides_0"), val = tensor([1])]; tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_58")]; - tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_278_split_sizes_0 = const()[name = tensor("op_278_split_sizes_0"), val = tensor([256, 256])]; - tensor var_278_axis_0 = const()[name = tensor("op_278_axis_0"), val = tensor(1)]; - tensor var_278_0, tensor var_278_1 = split(axis = var_278_axis_0, split_sizes = var_278_split_sizes_0, x = inputs_3)[name = tensor("op_278")]; - tensor var_280 = sigmoid(x = var_278_1)[name = tensor("op_280")]; - tensor inputs_5 = mul(x = var_278_0, y = var_280)[name = tensor("inputs_5")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; + tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; + tensor var_310_split_sizes_0 = const()[name = tensor("op_310_split_sizes_0"), val = tensor([256, 256])]; + tensor var_310_axis_0 = const()[name = tensor("op_310_axis_0"), val = tensor(1)]; + tensor var_310_0, tensor var_310_1 = split(axis = var_310_axis_0, split_sizes = var_310_split_sizes_0, x = inputs_3)[name = tensor("op_310")]; + tensor var_312 = sigmoid(x = var_310_1)[name = tensor("op_312")]; + tensor inputs_5 = mul(x = var_310_0, y = var_312)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; tensor outputs_aug_1_strides_0 = const()[name = tensor("outputs_aug_1_strides_0"), val = tensor([1])]; tensor outputs_aug_1_dilations_0 = const()[name = tensor("outputs_aug_1_dilations_0"), val = tensor([1])]; - tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; + tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; tensor input_25_begin_0 = const()[name = tensor("input_25_begin_0"), val = tensor([0, 0, 0])]; - tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([1, 256, 16])]; + tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([2, 256, 16])]; tensor input_25_end_mask_0 = const()[name = tensor("input_25_end_mask_0"), val = tensor([true, true, false])]; - tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; - tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; - tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; + tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; + tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; + tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; tensor input_29_pad_type_0 = const()[name = tensor("input_29_pad_type_0"), val = tensor("valid")]; tensor input_29_strides_0 = const()[name = tensor("input_29_strides_0"), val = tensor([1])]; tensor input_29_pad_0 = const()[name = tensor("input_29_pad_0"), val = tensor([0, 0])]; tensor input_29_dilations_0 = const()[name = tensor("input_29_dilations_0"), val = tensor([1])]; tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; - tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; + tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_311_begin_0 = const()[name = tensor("op_311_begin_0"), val = tensor([0, -1, 0])]; - tensor var_311_end_0 = const()[name = tensor("op_311_end_0"), val = tensor([1, 16, 256])]; - tensor var_311_end_mask_0 = const()[name = tensor("op_311_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_57")]; - tensor var_311 = slice_by_index(begin = var_311_begin_0, end = var_311_end_0, end_mask = var_311_end_mask_0, x = conv_out_1)[name = tensor("op_311")]; - tensor var_314_begin_0 = const()[name = tensor("op_314_begin_0"), val = tensor([0, 1, 0])]; - tensor var_314_end_0 = const()[name = tensor("op_314_end_0"), val = tensor([1, 1, 256])]; - tensor var_314_end_mask_0 = const()[name = tensor("op_314_end_mask_0"), val = tensor([true, true, true])]; - tensor var_314 = slice_by_index(begin = var_314_begin_0, end = var_314_end_0, end_mask = var_314_end_mask_0, x = x_3)[name = tensor("op_314")]; - tensor var_317_begin_0 = const()[name = tensor("op_317_begin_0"), val = tensor([0, 1, 0])]; - tensor var_317_end_0 = const()[name = tensor("op_317_end_0"), val = tensor([1, 16, 256])]; - tensor var_317_end_mask_0 = const()[name = tensor("op_317_end_mask_0"), val = tensor([true, true, true])]; - tensor var_317 = slice_by_index(begin = var_317_begin_0, end = var_317_end_0, end_mask = var_317_end_mask_0, x = input_21)[name = tensor("op_317")]; - tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_317, var_314))[name = tensor("rolled_3")]; - tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; - tensor var_322 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_322")]; - tensor var_323_dtype_0 = const()[name = tensor("op_323_dtype_0"), val = tensor("fp32")]; - tensor var_324 = const()[name = tensor("op_324"), val = tensor([1, 16, 1])]; - tensor var_323 = cast(dtype = var_323_dtype_0, x = var_322)[name = tensor("cast_141")]; - tensor fill_mask_3 = reshape(shape = var_324, x = var_323)[name = tensor("fill_mask_3")]; - tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_314)[name = tensor("x_expanded_3")]; - tensor var_328 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_328")]; - tensor var_329 = mul(x = input_21, y = var_328)[name = tensor("op_329")]; - tensor var_330 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_330")]; - tensor filled_window_3 = add(x = var_329, y = var_330)[name = tensor("filled_window_3")]; - tensor var_332 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_332")]; - tensor var_333_dtype_0 = const()[name = tensor("op_333_dtype_0"), val = tensor("fp32")]; - tensor var_334 = const()[name = tensor("op_334"), val = tensor([1, 1, 1])]; - tensor var_333 = cast(dtype = var_333_dtype_0, x = var_332)[name = tensor("cast_140")]; - tensor use_roll_3 = reshape(shape = var_334, x = var_333)[name = tensor("use_roll_3")]; - tensor var_336 = sub(x = var_17, y = use_roll_3)[name = tensor("op_336")]; - tensor var_337 = mul(x = filled_window_3, y = var_336)[name = tensor("op_337")]; - tensor var_338 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_338")]; - tensor input_31 = add(x = var_337, y = var_338)[name = tensor("input_31")]; - tensor var_340 = const()[name = tensor("op_340"), val = tensor(0x1p+0)]; - tensor var_341 = add(x = clip_1, y = var_340)[name = tensor("op_341")]; - tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_341)[name = tensor("clip_3")]; - tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; - tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_31)[name = tensor("x_7")]; - tensor input_33_perm_0 = const()[name = tensor("input_33_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_9_pad_type_0 = const()[name = tensor("inputs_9_pad_type_0"), val = tensor("valid")]; - tensor inputs_9_strides_0 = const()[name = tensor("inputs_9_strides_0"), val = tensor([1])]; - tensor inputs_9_pad_0 = const()[name = tensor("inputs_9_pad_0"), val = tensor([0, 0])]; - tensor inputs_9_dilations_0 = const()[name = tensor("inputs_9_dilations_0"), val = tensor([1])]; - tensor inputs_9_groups_0 = const()[name = tensor("inputs_9_groups_0"), val = tensor(1)]; - tensor input_33 = transpose(perm = input_33_perm_0, x = x_7)[name = tensor("transpose_56")]; - tensor inputs_9 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_9_dilations_0, groups = inputs_9_groups_0, pad = inputs_9_pad_0, pad_type = inputs_9_pad_type_0, strides = inputs_9_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_33)[name = tensor("inputs_9")]; - tensor var_362_split_sizes_0 = const()[name = tensor("op_362_split_sizes_0"), val = tensor([256, 256])]; - tensor var_362_axis_0 = const()[name = tensor("op_362_axis_0"), val = tensor(1)]; - tensor var_362_0, tensor var_362_1 = split(axis = var_362_axis_0, split_sizes = var_362_split_sizes_0, x = inputs_9)[name = tensor("op_362")]; - tensor var_364 = sigmoid(x = var_362_1)[name = tensor("op_364")]; - tensor inputs_11 = mul(x = var_362_0, y = var_364)[name = tensor("inputs_11")]; - tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; - tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; - tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; - tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_11)[name = tensor("outputs_aug_3")]; - tensor input_35_begin_0 = const()[name = tensor("input_35_begin_0"), val = tensor([0, 0, 0])]; - tensor input_35_end_0 = const()[name = tensor("input_35_end_0"), val = tensor([1, 256, 16])]; - tensor input_35_end_mask_0 = const()[name = tensor("input_35_end_mask_0"), val = tensor([true, true, false])]; - tensor input_35 = slice_by_index(begin = input_35_begin_0, end = input_35_end_0, end_mask = input_35_end_mask_0, x = outputs_aug_3)[name = tensor("input_35")]; - tensor inputs_13 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_35)[name = tensor("inputs_13")]; - tensor input_37 = silu(x = inputs_13)[name = tensor("input_37")]; - tensor input_39_pad_type_0 = const()[name = tensor("input_39_pad_type_0"), val = tensor("valid")]; - tensor input_39_strides_0 = const()[name = tensor("input_39_strides_0"), val = tensor([1])]; - tensor input_39_pad_0 = const()[name = tensor("input_39_pad_0"), val = tensor([0, 0])]; - tensor input_39_dilations_0 = const()[name = tensor("input_39_dilations_0"), val = tensor([1])]; - tensor input_39_groups_0 = const()[name = tensor("input_39_groups_0"), val = tensor(1)]; - tensor input_39 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_37)[name = tensor("input_39")]; - tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_395_begin_0 = const()[name = tensor("op_395_begin_0"), val = tensor([0, -1, 0])]; - tensor var_395_end_0 = const()[name = tensor("op_395_end_0"), val = tensor([1, 16, 256])]; - tensor var_395_end_mask_0 = const()[name = tensor("op_395_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_39)[name = tensor("transpose_55")]; - tensor var_395 = slice_by_index(begin = var_395_begin_0, end = var_395_end_0, end_mask = var_395_end_mask_0, x = conv_out_3)[name = tensor("op_395")]; - tensor var_398_interleave_0 = const()[name = tensor("op_398_interleave_0"), val = tensor(false)]; - tensor var_398 = concat(axis = var_33, interleave = var_398_interleave_0, values = (var_311, var_395))[name = tensor("op_398")]; - tensor input_41 = add(x = x_3, y = var_398)[name = tensor("input_41")]; + tensor var_343_begin_0 = const()[name = tensor("op_343_begin_0"), val = tensor([0, -1, 0])]; + tensor var_343_end_0 = const()[name = tensor("op_343_end_0"), val = tensor([2, 16, 256])]; + tensor var_343_end_mask_0 = const()[name = tensor("op_343_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_343 = slice_by_index(begin = var_343_begin_0, end = var_343_end_0, end_mask = var_343_end_mask_0, x = conv_out_1)[name = tensor("op_343")]; + tensor var_345_perm_0 = const()[name = tensor("op_345_perm_0"), val = tensor([1, 0, 2])]; + tensor var_345 = transpose(perm = var_345_perm_0, x = var_343)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_345)[name = tensor("input_31")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; + tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; + tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; + tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; + tensor var_368 = const()[name = tensor("op_368"), val = tensor(0x1p-1)]; + tensor var_369 = mul(x = input_39, y = var_368)[name = tensor("op_369")]; + tensor input_41 = add(x = var_369, y = input_31)[name = tensor("input_41")]; tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; - tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_41)[name = tensor("input_43")]; - tensor inputs_15 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_43)[name = tensor("linear_8")]; - tensor input_45 = silu(x = inputs_15)[name = tensor("input_45")]; - tensor input_49 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_45)[name = tensor("linear_9")]; - tensor var_421 = const()[name = tensor("op_421"), val = tensor(0x1p-1)]; - tensor var_422 = mul(x = input_49, y = var_421)[name = tensor("op_422")]; - tensor input_51 = add(x = var_422, y = input_41)[name = tensor("input_51")]; - tensor input_53_axes_0 = const()[name = tensor("input_53_axes_0"), val = tensor([-1])]; - tensor input_53 = layer_norm(axes = input_53_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_51)[name = tensor("input_53")]; - tensor input_55_axes_0 = const()[name = tensor("input_55_axes_0"), val = tensor([-1])]; - tensor input_55 = layer_norm(axes = input_55_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_53)[name = tensor("input_55")]; - tensor inputs_17 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_55)[name = tensor("linear_10")]; - tensor input_57 = silu(x = inputs_17)[name = tensor("input_57")]; - tensor input_61 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_57)[name = tensor("linear_11")]; - tensor var_451 = const()[name = tensor("op_451"), val = tensor(0x1p-1)]; - tensor var_452 = mul(x = input_61, y = var_451)[name = tensor("op_452")]; - tensor input_63 = add(x = var_452, y = input_53)[name = tensor("input_63")]; - tensor x_9_axes_0 = const()[name = tensor("x_9_axes_0"), val = tensor([-1])]; - tensor x_9 = layer_norm(axes = x_9_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_63)[name = tensor("x_9")]; + tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; + tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; + tensor input_45 = layer_norm(axes = input_45_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_43)[name = tensor("input_45")]; + tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; + tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; + tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; + tensor var_398 = const()[name = tensor("op_398"), val = tensor(0x1p-1)]; + tensor var_399 = mul(x = input_51, y = var_398)[name = tensor("op_399")]; + tensor input_53 = add(x = var_399, y = input_43)[name = tensor("input_53")]; + tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; + tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_3_end_0 = const()[name = tensor("prev_kv_3_end_0"), val = tensor([2, 1, 4, 64, 64])]; tensor prev_kv_3_end_mask_0 = const()[name = tensor("prev_kv_3_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -490,243 +454,206 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_466 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_9)[name = tensor("linear_12")]; - tensor var_467 = const()[name = tensor("op_467"), val = tensor([1, 2, 4, 64])]; - tensor var_468 = reshape(shape = var_467, x = var_466)[name = tensor("op_468")]; + tensor var_413 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_414 = const()[name = tensor("op_414"), val = tensor([1, 2, 4, 64])]; + tensor var_415 = reshape(shape = var_414, x = var_413)[name = tensor("op_415")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_472 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_9)[name = tensor("linear_13")]; - tensor var_473 = const()[name = tensor("op_473"), val = tensor(0x1p-3)]; - tensor var_474 = mul(x = var_472, y = var_473)[name = tensor("op_474")]; - tensor var_475 = const()[name = tensor("op_475"), val = tensor([1, 2, 4, 64])]; - tensor var_476 = reshape(shape = var_475, x = var_474)[name = tensor("op_476")]; + tensor var_419 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_420 = const()[name = tensor("op_420"), val = tensor(0x1p-3)]; + tensor var_421 = mul(x = var_419, y = var_420)[name = tensor("op_421")]; + tensor var_422 = const()[name = tensor("op_422"), val = tensor([1, 2, 4, 64])]; + tensor var_423 = reshape(shape = var_422, x = var_421)[name = tensor("op_423")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_480 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_9)[name = tensor("linear_14")]; - tensor var_481 = const()[name = tensor("op_481"), val = tensor([1, 2, 4, 64])]; - tensor var_482 = reshape(shape = var_481, x = var_480)[name = tensor("op_482")]; + tensor var_427 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_428 = const()[name = tensor("op_428"), val = tensor([1, 2, 4, 64])]; + tensor var_429 = reshape(shape = var_428, x = var_427)[name = tensor("op_429")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_67 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_9)[name = tensor("linear_15")]; + tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; tensor s_t_3 = add(x = prev_scale_3, y = encoder__t_index)[name = tensor("s_t_3")]; tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_476)[name = tensor("transpose_53")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_468)[name = tensor("transpose_54")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_423)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_415)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_492 = const()[name = tensor("op_492"), val = tensor([2, 1])]; - tensor var_493 = reshape(shape = var_492, x = sqrt_s_t_3)[name = tensor("op_493")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_493)[name = tensor("M_3")]; - tensor var_495 = mul(x = qk_3, y = M_3)[name = tensor("op_495")]; + tensor var_439 = const()[name = tensor("op_439"), val = tensor([2, 1])]; + tensor var_440 = reshape(shape = var_439, x = sqrt_s_t_3)[name = tensor("op_440")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_440)[name = tensor("M_3")]; + tensor var_442 = mul(x = qk_3, y = M_3)[name = tensor("op_442")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_482)[name = tensor("transpose_52")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_495, y = v_3)[name = tensor("inner_3")]; - tensor var_497_transpose_x_0 = const()[name = tensor("op_497_transpose_x_0"), val = tensor(false)]; - tensor var_497_transpose_y_0 = const()[name = tensor("op_497_transpose_y_0"), val = tensor(false)]; - tensor var_497 = matmul(transpose_x = var_497_transpose_x_0, transpose_y = var_497_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_497")]; - tensor var_498 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_498")]; - tensor var_499 = const()[name = tensor("op_499"), val = tensor([1, 1, 2, 1])]; - tensor var_500 = reshape(shape = var_499, x = var_498)[name = tensor("op_500")]; - tensor cross_3 = mul(x = var_497, y = var_500)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_429)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_442, y = v_3)[name = tensor("inner_3")]; + tensor var_444_transpose_x_0 = const()[name = tensor("op_444_transpose_x_0"), val = tensor(false)]; + tensor var_444_transpose_y_0 = const()[name = tensor("op_444_transpose_y_0"), val = tensor(false)]; + tensor var_444 = matmul(transpose_x = var_444_transpose_x_0, transpose_y = var_444_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_444")]; + tensor var_445 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_445")]; + tensor var_446 = const()[name = tensor("op_446"), val = tensor([1, 1, 2, 1])]; + tensor var_447 = reshape(shape = var_446, x = var_445)[name = tensor("op_447")]; + tensor cross_3 = mul(x = var_444, y = var_447)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_503 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_503")]; - tensor var_505_transpose_x_1 = const()[name = tensor("op_505_transpose_x_1"), val = tensor(true)]; - tensor var_505_transpose_y_1 = const()[name = tensor("op_505_transpose_y_1"), val = tensor(false)]; - tensor var_505 = matmul(transpose_x = var_505_transpose_x_1, transpose_y = var_505_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_505")]; - tensor new_kv_unnorm_3 = add(x = var_503, y = var_505)[name = tensor("new_kv_unnorm_3")]; - tensor var_507 = const()[name = tensor("op_507"), val = tensor(0x1p+1)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_507)[name = tensor("new_scale_3")]; - tensor var_509 = sqrt(x = new_scale_3)[name = tensor("op_509")]; - tensor var_510 = real_div(x = new_kv_unnorm_3, y = var_509)[name = tensor("op_510")]; - tensor var_511_perm_0 = const()[name = tensor("op_511_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_450 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_450")]; + tensor var_452_transpose_x_1 = const()[name = tensor("op_452_transpose_x_1"), val = tensor(true)]; + tensor var_452_transpose_y_1 = const()[name = tensor("op_452_transpose_y_1"), val = tensor(false)]; + tensor var_452 = matmul(transpose_x = var_452_transpose_x_1, transpose_y = var_452_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_452")]; + tensor new_kv_unnorm_3 = add(x = var_450, y = var_452)[name = tensor("new_kv_unnorm_3")]; + tensor var_454 = const()[name = tensor("op_454"), val = tensor(0x1p+1)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_454)[name = tensor("new_scale_3")]; + tensor var_456 = sqrt(x = new_scale_3)[name = tensor("op_456")]; + tensor var_457 = real_div(x = new_kv_unnorm_3, y = var_456)[name = tensor("op_457")]; + tensor var_458_perm_0 = const()[name = tensor("op_458_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_511 = transpose(perm = var_511_perm_0, x = out_7)[name = tensor("transpose_51")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_511)[name = tensor("out_9")]; - tensor var_515 = const()[name = tensor("op_515"), val = tensor([1, 2, 256])]; - tensor out_11 = reshape(shape = var_515, x = out_9)[name = tensor("out_11")]; - tensor var_517 = silu(x = input_67)[name = tensor("op_517")]; - tensor input_69 = mul(x = var_517, y = out_11)[name = tensor("input_69")]; - tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_69)[name = tensor("linear_16")]; - tensor x_11 = add(x = input_63, y = ret_out_3)[name = tensor("x_11")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor var_458 = transpose(perm = var_458_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_458)[name = tensor("out_9")]; + tensor var_462 = const()[name = tensor("op_462"), val = tensor([1, 2, 256])]; + tensor out_11 = reshape(shape = var_462, x = out_9)[name = tensor("out_11")]; + tensor var_464 = silu(x = input_57)[name = tensor("op_464")]; + tensor input_59 = mul(x = var_464, y = out_11)[name = tensor("input_59")]; + tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; + tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; + tensor window_7_begin_0 = const()[name = tensor("window_7_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_7_end_0 = const()[name = tensor("window_7_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_7_end_mask_0 = const()[name = tensor("window_7_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_7_squeeze_mask_0 = const()[name = tensor("window_7_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_7 = slice_by_index(begin = window_7_begin_0, end = window_7_end_0, end_mask = window_7_end_mask_0, squeeze_mask = window_7_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_7")]; tensor filled_5_begin_0 = const()[name = tensor("filled_5_begin_0"), val = tensor([1, 0])]; tensor filled_5_end_0 = const()[name = tensor("filled_5_end_0"), val = tensor([2, 1])]; tensor filled_5_end_mask_0 = const()[name = tensor("filled_5_end_mask_0"), val = tensor([false, true])]; tensor filled_5_squeeze_mask_0 = const()[name = tensor("filled_5_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_5 = slice_by_index(begin = filled_5_begin_0, end = filled_5_end_0, end_mask = filled_5_end_mask_0, squeeze_mask = filled_5_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_5")]; - tensor var_526_begin_0 = const()[name = tensor("op_526_begin_0"), val = tensor([0, 0, 0])]; - tensor var_526_end_0 = const()[name = tensor("op_526_end_0"), val = tensor([1, 1, 256])]; - tensor var_526_end_mask_0 = const()[name = tensor("op_526_end_mask_0"), val = tensor([true, false, true])]; - tensor var_526 = slice_by_index(begin = var_526_begin_0, end = var_526_end_0, end_mask = var_526_end_mask_0, x = x_11)[name = tensor("op_526")]; - tensor var_529_begin_0 = const()[name = tensor("op_529_begin_0"), val = tensor([0, 1, 0])]; - tensor var_529_end_0 = const()[name = tensor("op_529_end_0"), val = tensor([1, 16, 256])]; - tensor var_529_end_mask_0 = const()[name = tensor("op_529_end_mask_0"), val = tensor([true, true, true])]; - tensor var_529 = slice_by_index(begin = var_529_begin_0, end = var_529_end_0, end_mask = var_529_end_mask_0, x = window_3)[name = tensor("op_529")]; + tensor var_473_begin_0 = const()[name = tensor("op_473_begin_0"), val = tensor([0, 0, 0])]; + tensor var_473_end_0 = const()[name = tensor("op_473_end_0"), val = tensor([1, 1, 256])]; + tensor var_473_end_mask_0 = const()[name = tensor("op_473_end_mask_0"), val = tensor([true, false, true])]; + tensor var_473 = slice_by_index(begin = var_473_begin_0, end = var_473_end_0, end_mask = var_473_end_mask_0, x = x_9)[name = tensor("op_473")]; + tensor var_476_begin_0 = const()[name = tensor("op_476_begin_0"), val = tensor([0, 1, 0])]; + tensor var_476_end_0 = const()[name = tensor("op_476_end_0"), val = tensor([1, 16, 256])]; + tensor var_476_end_mask_0 = const()[name = tensor("op_476_end_mask_0"), val = tensor([true, true, true])]; + tensor var_476 = slice_by_index(begin = var_476_begin_0, end = var_476_end_0, end_mask = var_476_end_mask_0, x = window_7)[name = tensor("op_476")]; tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_529, var_526))[name = tensor("rolled_5")]; + tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_476, var_473))[name = tensor("rolled_5")]; tensor clip_4 = clip(alpha = var_21, beta = var_20, x = filled_5)[name = tensor("clip_4")]; - tensor var_534 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_534")]; - tensor var_535_dtype_0 = const()[name = tensor("op_535_dtype_0"), val = tensor("fp32")]; - tensor var_536 = const()[name = tensor("op_536"), val = tensor([1, 16, 1])]; - tensor var_535 = cast(dtype = var_535_dtype_0, x = var_534)[name = tensor("cast_139")]; - tensor fill_mask_5 = reshape(shape = var_536, x = var_535)[name = tensor("fill_mask_5")]; + tensor var_481 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_481")]; + tensor var_482_dtype_0 = const()[name = tensor("op_482_dtype_0"), val = tensor("fp32")]; + tensor var_483 = const()[name = tensor("op_483"), val = tensor([1, 16, 1])]; + tensor var_482 = cast(dtype = var_482_dtype_0, x = var_481)[name = tensor("cast_139")]; + tensor fill_mask_5 = reshape(shape = var_483, x = var_482)[name = tensor("fill_mask_5")]; tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_526)[name = tensor("x_expanded_5")]; - tensor var_540 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_540")]; - tensor var_541 = mul(x = window_3, y = var_540)[name = tensor("op_541")]; - tensor var_542 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_542")]; - tensor filled_window_5 = add(x = var_541, y = var_542)[name = tensor("filled_window_5")]; - tensor var_544 = greater_equal(x = filled_5, y = var_16)[name = tensor("op_544")]; - tensor var_545_dtype_0 = const()[name = tensor("op_545_dtype_0"), val = tensor("fp32")]; - tensor var_546 = const()[name = tensor("op_546"), val = tensor([1, 1, 1])]; - tensor var_545 = cast(dtype = var_545_dtype_0, x = var_544)[name = tensor("cast_138")]; - tensor use_roll_5 = reshape(shape = var_546, x = var_545)[name = tensor("use_roll_5")]; - tensor var_548 = sub(x = var_17, y = use_roll_5)[name = tensor("op_548")]; - tensor var_549 = mul(x = filled_window_5, y = var_548)[name = tensor("op_549")]; - tensor var_550 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_550")]; - tensor input_71 = add(x = var_549, y = var_550)[name = tensor("input_71")]; - tensor var_552 = const()[name = tensor("op_552"), val = tensor(0x1p+0)]; - tensor var_553 = add(x = filled_5, y = var_552)[name = tensor("op_553")]; - tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_553)[name = tensor("clip_5")]; - tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; - tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_71)[name = tensor("x_13")]; - tensor input_73_perm_0 = const()[name = tensor("input_73_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_19_pad_type_0 = const()[name = tensor("inputs_19_pad_type_0"), val = tensor("valid")]; - tensor inputs_19_strides_0 = const()[name = tensor("inputs_19_strides_0"), val = tensor([1])]; - tensor inputs_19_pad_0 = const()[name = tensor("inputs_19_pad_0"), val = tensor([0, 0])]; - tensor inputs_19_dilations_0 = const()[name = tensor("inputs_19_dilations_0"), val = tensor([1])]; - tensor inputs_19_groups_0 = const()[name = tensor("inputs_19_groups_0"), val = tensor(1)]; - tensor input_73 = transpose(perm = input_73_perm_0, x = x_13)[name = tensor("transpose_50")]; - tensor inputs_19 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_19_dilations_0, groups = inputs_19_groups_0, pad = inputs_19_pad_0, pad_type = inputs_19_pad_type_0, strides = inputs_19_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_73)[name = tensor("inputs_19")]; - tensor var_574_split_sizes_0 = const()[name = tensor("op_574_split_sizes_0"), val = tensor([256, 256])]; - tensor var_574_axis_0 = const()[name = tensor("op_574_axis_0"), val = tensor(1)]; - tensor var_574_0, tensor var_574_1 = split(axis = var_574_axis_0, split_sizes = var_574_split_sizes_0, x = inputs_19)[name = tensor("op_574")]; - tensor var_576 = sigmoid(x = var_574_1)[name = tensor("op_576")]; - tensor inputs_21 = mul(x = var_574_0, y = var_576)[name = tensor("inputs_21")]; - tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; - tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; - tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; - tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_21)[name = tensor("outputs_aug_5")]; - tensor input_75_begin_0 = const()[name = tensor("input_75_begin_0"), val = tensor([0, 0, 0])]; - tensor input_75_end_0 = const()[name = tensor("input_75_end_0"), val = tensor([1, 256, 16])]; - tensor input_75_end_mask_0 = const()[name = tensor("input_75_end_mask_0"), val = tensor([true, true, false])]; - tensor input_75 = slice_by_index(begin = input_75_begin_0, end = input_75_end_0, end_mask = input_75_end_mask_0, x = outputs_aug_5)[name = tensor("input_75")]; - tensor inputs_23 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_75)[name = tensor("inputs_23")]; - tensor input_77 = silu(x = inputs_23)[name = tensor("input_77")]; - tensor input_79_pad_type_0 = const()[name = tensor("input_79_pad_type_0"), val = tensor("valid")]; - tensor input_79_strides_0 = const()[name = tensor("input_79_strides_0"), val = tensor([1])]; - tensor input_79_pad_0 = const()[name = tensor("input_79_pad_0"), val = tensor([0, 0])]; - tensor input_79_dilations_0 = const()[name = tensor("input_79_dilations_0"), val = tensor([1])]; - tensor input_79_groups_0 = const()[name = tensor("input_79_groups_0"), val = tensor(1)]; - tensor input_79 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_79_dilations_0, groups = input_79_groups_0, pad = input_79_pad_0, pad_type = input_79_pad_type_0, strides = input_79_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_77)[name = tensor("input_79")]; - tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_607_begin_0 = const()[name = tensor("op_607_begin_0"), val = tensor([0, -1, 0])]; - tensor var_607_end_0 = const()[name = tensor("op_607_end_0"), val = tensor([1, 16, 256])]; - tensor var_607_end_mask_0 = const()[name = tensor("op_607_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_79)[name = tensor("transpose_49")]; - tensor var_607 = slice_by_index(begin = var_607_begin_0, end = var_607_end_0, end_mask = var_607_end_mask_0, x = conv_out_5)[name = tensor("op_607")]; - tensor var_610_begin_0 = const()[name = tensor("op_610_begin_0"), val = tensor([0, 1, 0])]; - tensor var_610_end_0 = const()[name = tensor("op_610_end_0"), val = tensor([1, 1, 256])]; - tensor var_610_end_mask_0 = const()[name = tensor("op_610_end_mask_0"), val = tensor([true, true, true])]; - tensor var_610 = slice_by_index(begin = var_610_begin_0, end = var_610_end_0, end_mask = var_610_end_mask_0, x = x_11)[name = tensor("op_610")]; - tensor var_613_begin_0 = const()[name = tensor("op_613_begin_0"), val = tensor([0, 1, 0])]; - tensor var_613_end_0 = const()[name = tensor("op_613_end_0"), val = tensor([1, 16, 256])]; - tensor var_613_end_mask_0 = const()[name = tensor("op_613_end_mask_0"), val = tensor([true, true, true])]; - tensor var_613 = slice_by_index(begin = var_613_begin_0, end = var_613_end_0, end_mask = var_613_end_mask_0, x = input_71)[name = tensor("op_613")]; + tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_473)[name = tensor("x_expanded_5")]; + tensor var_487 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_487")]; + tensor var_488 = mul(x = window_7, y = var_487)[name = tensor("op_488")]; + tensor var_489 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_489")]; + tensor filled_window_5 = add(x = var_488, y = var_489)[name = tensor("filled_window_5")]; + tensor var_491 = greater_equal(x = filled_5, y = var_16)[name = tensor("op_491")]; + tensor var_492_dtype_0 = const()[name = tensor("op_492_dtype_0"), val = tensor("fp32")]; + tensor var_493 = const()[name = tensor("op_493"), val = tensor([1, 1, 1])]; + tensor var_492 = cast(dtype = var_492_dtype_0, x = var_491)[name = tensor("cast_138")]; + tensor use_roll_5 = reshape(shape = var_493, x = var_492)[name = tensor("use_roll_5")]; + tensor var_495 = sub(x = var_17, y = use_roll_5)[name = tensor("op_495")]; + tensor var_496 = mul(x = filled_window_5, y = var_495)[name = tensor("op_496")]; + tensor var_497 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_497")]; + tensor window_9 = add(x = var_496, y = var_497)[name = tensor("window_9")]; + tensor var_499 = const()[name = tensor("op_499"), val = tensor(0x1p+0)]; + tensor var_500 = add(x = filled_5, y = var_499)[name = tensor("op_500")]; + tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_500)[name = tensor("clip_5")]; + tensor var_503_begin_0 = const()[name = tensor("op_503_begin_0"), val = tensor([0, 1, 0])]; + tensor var_503_end_0 = const()[name = tensor("op_503_end_0"), val = tensor([1, 1, 256])]; + tensor var_503_end_mask_0 = const()[name = tensor("op_503_end_mask_0"), val = tensor([true, true, true])]; + tensor var_503 = slice_by_index(begin = var_503_begin_0, end = var_503_end_0, end_mask = var_503_end_mask_0, x = x_9)[name = tensor("op_503")]; + tensor var_506_begin_0 = const()[name = tensor("op_506_begin_0"), val = tensor([0, 1, 0])]; + tensor var_506_end_0 = const()[name = tensor("op_506_end_0"), val = tensor([1, 16, 256])]; + tensor var_506_end_mask_0 = const()[name = tensor("op_506_end_mask_0"), val = tensor([true, true, true])]; + tensor var_506 = slice_by_index(begin = var_506_begin_0, end = var_506_end_0, end_mask = var_506_end_mask_0, x = window_9)[name = tensor("op_506")]; tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; - tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_613, var_610))[name = tensor("rolled_7")]; + tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_506, var_503))[name = tensor("rolled_7")]; tensor clip_6 = clip(alpha = var_21, beta = var_20, x = clip_5)[name = tensor("clip_6")]; - tensor var_618 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_618")]; - tensor var_619_dtype_0 = const()[name = tensor("op_619_dtype_0"), val = tensor("fp32")]; - tensor var_620 = const()[name = tensor("op_620"), val = tensor([1, 16, 1])]; - tensor var_619 = cast(dtype = var_619_dtype_0, x = var_618)[name = tensor("cast_137")]; - tensor fill_mask_7 = reshape(shape = var_620, x = var_619)[name = tensor("fill_mask_7")]; + tensor var_511 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_511")]; + tensor var_512_dtype_0 = const()[name = tensor("op_512_dtype_0"), val = tensor("fp32")]; + tensor var_513 = const()[name = tensor("op_513"), val = tensor([1, 16, 1])]; + tensor var_512 = cast(dtype = var_512_dtype_0, x = var_511)[name = tensor("cast_137")]; + tensor fill_mask_7 = reshape(shape = var_513, x = var_512)[name = tensor("fill_mask_7")]; tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_610)[name = tensor("x_expanded_7")]; - tensor var_624 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_624")]; - tensor var_625 = mul(x = input_71, y = var_624)[name = tensor("op_625")]; - tensor var_626 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_626")]; - tensor filled_window_7 = add(x = var_625, y = var_626)[name = tensor("filled_window_7")]; - tensor var_628 = greater_equal(x = clip_5, y = var_16)[name = tensor("op_628")]; - tensor var_629_dtype_0 = const()[name = tensor("op_629_dtype_0"), val = tensor("fp32")]; - tensor var_630 = const()[name = tensor("op_630"), val = tensor([1, 1, 1])]; - tensor var_629 = cast(dtype = var_629_dtype_0, x = var_628)[name = tensor("cast_136")]; - tensor use_roll_7 = reshape(shape = var_630, x = var_629)[name = tensor("use_roll_7")]; - tensor var_632 = sub(x = var_17, y = use_roll_7)[name = tensor("op_632")]; - tensor var_633 = mul(x = filled_window_7, y = var_632)[name = tensor("op_633")]; - tensor var_634 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_634")]; - tensor input_81 = add(x = var_633, y = var_634)[name = tensor("input_81")]; - tensor var_636 = const()[name = tensor("op_636"), val = tensor(0x1p+0)]; - tensor var_637 = add(x = clip_5, y = var_636)[name = tensor("op_637")]; - tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_637)[name = tensor("clip_7")]; - tensor x_15_axes_0 = const()[name = tensor("x_15_axes_0"), val = tensor([-1])]; - tensor x_15 = layer_norm(axes = x_15_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_81)[name = tensor("x_15")]; - tensor input_83_perm_0 = const()[name = tensor("input_83_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_25_pad_type_0 = const()[name = tensor("inputs_25_pad_type_0"), val = tensor("valid")]; - tensor inputs_25_strides_0 = const()[name = tensor("inputs_25_strides_0"), val = tensor([1])]; - tensor inputs_25_pad_0 = const()[name = tensor("inputs_25_pad_0"), val = tensor([0, 0])]; - tensor inputs_25_dilations_0 = const()[name = tensor("inputs_25_dilations_0"), val = tensor([1])]; - tensor inputs_25_groups_0 = const()[name = tensor("inputs_25_groups_0"), val = tensor(1)]; - tensor input_83 = transpose(perm = input_83_perm_0, x = x_15)[name = tensor("transpose_48")]; - tensor inputs_25 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_25_dilations_0, groups = inputs_25_groups_0, pad = inputs_25_pad_0, pad_type = inputs_25_pad_type_0, strides = inputs_25_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_83)[name = tensor("inputs_25")]; - tensor var_658_split_sizes_0 = const()[name = tensor("op_658_split_sizes_0"), val = tensor([256, 256])]; - tensor var_658_axis_0 = const()[name = tensor("op_658_axis_0"), val = tensor(1)]; - tensor var_658_0, tensor var_658_1 = split(axis = var_658_axis_0, split_sizes = var_658_split_sizes_0, x = inputs_25)[name = tensor("op_658")]; - tensor var_660 = sigmoid(x = var_658_1)[name = tensor("op_660")]; - tensor inputs_27 = mul(x = var_658_0, y = var_660)[name = tensor("inputs_27")]; - tensor outputs_aug_7_pad_type_0 = const()[name = tensor("outputs_aug_7_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_7_pad_0 = const()[name = tensor("outputs_aug_7_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_7_groups_0 = const()[name = tensor("outputs_aug_7_groups_0"), val = tensor(256)]; - tensor outputs_aug_7_strides_0 = const()[name = tensor("outputs_aug_7_strides_0"), val = tensor([1])]; - tensor outputs_aug_7_dilations_0 = const()[name = tensor("outputs_aug_7_dilations_0"), val = tensor([1])]; - tensor outputs_aug_7 = conv(dilations = outputs_aug_7_dilations_0, groups = outputs_aug_7_groups_0, pad = outputs_aug_7_pad_0, pad_type = outputs_aug_7_pad_type_0, strides = outputs_aug_7_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_27)[name = tensor("outputs_aug_7")]; - tensor input_85_begin_0 = const()[name = tensor("input_85_begin_0"), val = tensor([0, 0, 0])]; - tensor input_85_end_0 = const()[name = tensor("input_85_end_0"), val = tensor([1, 256, 16])]; - tensor input_85_end_mask_0 = const()[name = tensor("input_85_end_mask_0"), val = tensor([true, true, false])]; - tensor input_85 = slice_by_index(begin = input_85_begin_0, end = input_85_end_0, end_mask = input_85_end_mask_0, x = outputs_aug_7)[name = tensor("input_85")]; - tensor inputs_29 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_85)[name = tensor("inputs_29")]; - tensor input_87 = silu(x = inputs_29)[name = tensor("input_87")]; - tensor input_89_pad_type_0 = const()[name = tensor("input_89_pad_type_0"), val = tensor("valid")]; - tensor input_89_strides_0 = const()[name = tensor("input_89_strides_0"), val = tensor([1])]; - tensor input_89_pad_0 = const()[name = tensor("input_89_pad_0"), val = tensor([0, 0])]; - tensor input_89_dilations_0 = const()[name = tensor("input_89_dilations_0"), val = tensor([1])]; - tensor input_89_groups_0 = const()[name = tensor("input_89_groups_0"), val = tensor(1)]; - tensor input_89 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_89_dilations_0, groups = input_89_groups_0, pad = input_89_pad_0, pad_type = input_89_pad_type_0, strides = input_89_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_87)[name = tensor("input_89")]; - tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_691_begin_0 = const()[name = tensor("op_691_begin_0"), val = tensor([0, -1, 0])]; - tensor var_691_end_0 = const()[name = tensor("op_691_end_0"), val = tensor([1, 16, 256])]; - tensor var_691_end_mask_0 = const()[name = tensor("op_691_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_89)[name = tensor("transpose_47")]; - tensor var_691 = slice_by_index(begin = var_691_begin_0, end = var_691_end_0, end_mask = var_691_end_mask_0, x = conv_out_7)[name = tensor("op_691")]; - tensor var_694_interleave_0 = const()[name = tensor("op_694_interleave_0"), val = tensor(false)]; - tensor var_694 = concat(axis = var_33, interleave = var_694_interleave_0, values = (var_607, var_691))[name = tensor("op_694")]; - tensor input_91 = add(x = x_11, y = var_694)[name = tensor("input_91")]; - tensor input_93_axes_0 = const()[name = tensor("input_93_axes_0"), val = tensor([-1])]; - tensor input_93 = layer_norm(axes = input_93_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_91)[name = tensor("input_93")]; - tensor inputs_31 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_93)[name = tensor("linear_17")]; - tensor input_95 = silu(x = inputs_31)[name = tensor("input_95")]; - tensor input_99 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_95)[name = tensor("linear_18")]; - tensor var_717 = const()[name = tensor("op_717"), val = tensor(0x1p-1)]; - tensor var_718 = mul(x = input_99, y = var_717)[name = tensor("op_718")]; - tensor input_101 = add(x = var_718, y = input_91)[name = tensor("input_101")]; - tensor input_103_axes_0 = const()[name = tensor("input_103_axes_0"), val = tensor([-1])]; - tensor input_103 = layer_norm(axes = input_103_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_101)[name = tensor("input_103")]; - tensor input_105_axes_0 = const()[name = tensor("input_105_axes_0"), val = tensor([-1])]; - tensor input_105 = layer_norm(axes = input_105_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_103)[name = tensor("input_105")]; - tensor inputs_33 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_105)[name = tensor("linear_19")]; - tensor input_107 = silu(x = inputs_33)[name = tensor("input_107")]; - tensor input_111 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_107)[name = tensor("linear_20")]; - tensor var_747 = const()[name = tensor("op_747"), val = tensor(0x1p-1)]; - tensor var_748 = mul(x = input_111, y = var_747)[name = tensor("op_748")]; - tensor input_113 = add(x = var_748, y = input_103)[name = tensor("input_113")]; - tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; - tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_113)[name = tensor("x_17")]; + tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_503)[name = tensor("x_expanded_7")]; + tensor var_517 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_517")]; + tensor var_518 = mul(x = window_9, y = var_517)[name = tensor("op_518")]; + tensor var_519 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_519")]; + tensor filled_window_7 = add(x = var_518, y = var_519)[name = tensor("filled_window_7")]; + tensor var_521 = greater_equal(x = clip_5, y = var_16)[name = tensor("op_521")]; + tensor var_522_dtype_0 = const()[name = tensor("op_522_dtype_0"), val = tensor("fp32")]; + tensor var_523 = const()[name = tensor("op_523"), val = tensor([1, 1, 1])]; + tensor var_522 = cast(dtype = var_522_dtype_0, x = var_521)[name = tensor("cast_136")]; + tensor use_roll_7 = reshape(shape = var_523, x = var_522)[name = tensor("use_roll_7")]; + tensor var_525 = sub(x = var_17, y = use_roll_7)[name = tensor("op_525")]; + tensor var_526 = mul(x = filled_window_7, y = var_525)[name = tensor("op_526")]; + tensor var_527 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_527")]; + tensor window_11 = add(x = var_526, y = var_527)[name = tensor("window_11")]; + tensor var_529 = const()[name = tensor("op_529"), val = tensor(0x1p+0)]; + tensor var_530 = add(x = clip_5, y = var_529)[name = tensor("op_530")]; + tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_530)[name = tensor("clip_7")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_30, interleave = input_61_interleave_0, values = (window_9, window_11))[name = tensor("input_61")]; + tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; + tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; + tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_13_pad_type_0 = const()[name = tensor("inputs_13_pad_type_0"), val = tensor("valid")]; + tensor inputs_13_strides_0 = const()[name = tensor("inputs_13_strides_0"), val = tensor([1])]; + tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; + tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; + tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; + tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; + tensor var_553_split_sizes_0 = const()[name = tensor("op_553_split_sizes_0"), val = tensor([256, 256])]; + tensor var_553_axis_0 = const()[name = tensor("op_553_axis_0"), val = tensor(1)]; + tensor var_553_0, tensor var_553_1 = split(axis = var_553_axis_0, split_sizes = var_553_split_sizes_0, x = inputs_13)[name = tensor("op_553")]; + tensor var_555 = sigmoid(x = var_553_1)[name = tensor("op_555")]; + tensor inputs_15 = mul(x = var_553_0, y = var_555)[name = tensor("inputs_15")]; + tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; + tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; + tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; + tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_15)[name = tensor("outputs_aug_3")]; + tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; + tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([2, 256, 16])]; + tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; + tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_3)[name = tensor("input_65")]; + tensor inputs_17 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_65)[name = tensor("inputs_17")]; + tensor input_67 = silu(x = inputs_17)[name = tensor("input_67")]; + tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; + tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; + tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; + tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; + tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; + tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; + tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; + tensor var_586_begin_0 = const()[name = tensor("op_586_begin_0"), val = tensor([0, -1, 0])]; + tensor var_586_end_0 = const()[name = tensor("op_586_end_0"), val = tensor([2, 16, 256])]; + tensor var_586_end_mask_0 = const()[name = tensor("op_586_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_586 = slice_by_index(begin = var_586_begin_0, end = var_586_end_0, end_mask = var_586_end_mask_0, x = conv_out_3)[name = tensor("op_586")]; + tensor var_588_perm_0 = const()[name = tensor("op_588_perm_0"), val = tensor([1, 0, 2])]; + tensor var_588 = transpose(perm = var_588_perm_0, x = var_586)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_588)[name = tensor("input_71")]; + tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; + tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; + tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; + tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; + tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; + tensor var_611 = const()[name = tensor("op_611"), val = tensor(0x1p-1)]; + tensor var_612 = mul(x = input_79, y = var_611)[name = tensor("op_612")]; + tensor input_81 = add(x = var_612, y = input_71)[name = tensor("input_81")]; + tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; + tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; + tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; + tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; + tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; + tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; + tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; + tensor var_641 = const()[name = tensor("op_641"), val = tensor(0x1p-1)]; + tensor var_642 = mul(x = input_91, y = var_641)[name = tensor("op_642")]; + tensor input_93 = add(x = var_642, y = input_83)[name = tensor("input_93")]; + tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; + tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; tensor prev_kv_5_end_0 = const()[name = tensor("prev_kv_5_end_0"), val = tensor([3, 1, 4, 64, 64])]; tensor prev_kv_5_end_mask_0 = const()[name = tensor("prev_kv_5_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -737,243 +664,206 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_762 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_17)[name = tensor("linear_21")]; - tensor var_763 = const()[name = tensor("op_763"), val = tensor([1, 2, 4, 64])]; - tensor var_764 = reshape(shape = var_763, x = var_762)[name = tensor("op_764")]; + tensor var_656 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_657 = const()[name = tensor("op_657"), val = tensor([1, 2, 4, 64])]; + tensor var_658 = reshape(shape = var_657, x = var_656)[name = tensor("op_658")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_768 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_17)[name = tensor("linear_22")]; - tensor var_769 = const()[name = tensor("op_769"), val = tensor(0x1p-3)]; - tensor var_770 = mul(x = var_768, y = var_769)[name = tensor("op_770")]; - tensor var_771 = const()[name = tensor("op_771"), val = tensor([1, 2, 4, 64])]; - tensor var_772 = reshape(shape = var_771, x = var_770)[name = tensor("op_772")]; + tensor var_662 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_663 = const()[name = tensor("op_663"), val = tensor(0x1p-3)]; + tensor var_664 = mul(x = var_662, y = var_663)[name = tensor("op_664")]; + tensor var_665 = const()[name = tensor("op_665"), val = tensor([1, 2, 4, 64])]; + tensor var_666 = reshape(shape = var_665, x = var_664)[name = tensor("op_666")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_776 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_17)[name = tensor("linear_23")]; - tensor var_777 = const()[name = tensor("op_777"), val = tensor([1, 2, 4, 64])]; - tensor var_778 = reshape(shape = var_777, x = var_776)[name = tensor("op_778")]; + tensor var_670 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_671 = const()[name = tensor("op_671"), val = tensor([1, 2, 4, 64])]; + tensor var_672 = reshape(shape = var_671, x = var_670)[name = tensor("op_672")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_117 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_17)[name = tensor("linear_24")]; + tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; tensor s_t_5 = add(x = prev_scale_5, y = encoder__t_index)[name = tensor("s_t_5")]; tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_772)[name = tensor("transpose_45")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_764)[name = tensor("transpose_46")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_666)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_658)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_788 = const()[name = tensor("op_788"), val = tensor([2, 1])]; - tensor var_789 = reshape(shape = var_788, x = sqrt_s_t_5)[name = tensor("op_789")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_789)[name = tensor("M_5")]; - tensor var_791 = mul(x = qk_5, y = M_5)[name = tensor("op_791")]; + tensor var_682 = const()[name = tensor("op_682"), val = tensor([2, 1])]; + tensor var_683 = reshape(shape = var_682, x = sqrt_s_t_5)[name = tensor("op_683")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_683)[name = tensor("M_5")]; + tensor var_685 = mul(x = qk_5, y = M_5)[name = tensor("op_685")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_778)[name = tensor("transpose_44")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_791, y = v_5)[name = tensor("inner_5")]; - tensor var_793_transpose_x_0 = const()[name = tensor("op_793_transpose_x_0"), val = tensor(false)]; - tensor var_793_transpose_y_0 = const()[name = tensor("op_793_transpose_y_0"), val = tensor(false)]; - tensor var_793 = matmul(transpose_x = var_793_transpose_x_0, transpose_y = var_793_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_793")]; - tensor var_794 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_794")]; - tensor var_795 = const()[name = tensor("op_795"), val = tensor([1, 1, 2, 1])]; - tensor var_796 = reshape(shape = var_795, x = var_794)[name = tensor("op_796")]; - tensor cross_5 = mul(x = var_793, y = var_796)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_672)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_685, y = v_5)[name = tensor("inner_5")]; + tensor var_687_transpose_x_0 = const()[name = tensor("op_687_transpose_x_0"), val = tensor(false)]; + tensor var_687_transpose_y_0 = const()[name = tensor("op_687_transpose_y_0"), val = tensor(false)]; + tensor var_687 = matmul(transpose_x = var_687_transpose_x_0, transpose_y = var_687_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_687")]; + tensor var_688 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_688")]; + tensor var_689 = const()[name = tensor("op_689"), val = tensor([1, 1, 2, 1])]; + tensor var_690 = reshape(shape = var_689, x = var_688)[name = tensor("op_690")]; + tensor cross_5 = mul(x = var_687, y = var_690)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_799 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_799")]; - tensor var_801_transpose_x_1 = const()[name = tensor("op_801_transpose_x_1"), val = tensor(true)]; - tensor var_801_transpose_y_1 = const()[name = tensor("op_801_transpose_y_1"), val = tensor(false)]; - tensor var_801 = matmul(transpose_x = var_801_transpose_x_1, transpose_y = var_801_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_801")]; - tensor new_kv_unnorm_5 = add(x = var_799, y = var_801)[name = tensor("new_kv_unnorm_5")]; - tensor var_803 = const()[name = tensor("op_803"), val = tensor(0x1p+1)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_803)[name = tensor("new_scale_5")]; - tensor var_805 = sqrt(x = new_scale_5)[name = tensor("op_805")]; - tensor var_806 = real_div(x = new_kv_unnorm_5, y = var_805)[name = tensor("op_806")]; - tensor var_807_perm_0 = const()[name = tensor("op_807_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_693 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_693")]; + tensor var_695_transpose_x_1 = const()[name = tensor("op_695_transpose_x_1"), val = tensor(true)]; + tensor var_695_transpose_y_1 = const()[name = tensor("op_695_transpose_y_1"), val = tensor(false)]; + tensor var_695 = matmul(transpose_x = var_695_transpose_x_1, transpose_y = var_695_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_695")]; + tensor new_kv_unnorm_5 = add(x = var_693, y = var_695)[name = tensor("new_kv_unnorm_5")]; + tensor var_697 = const()[name = tensor("op_697"), val = tensor(0x1p+1)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_697)[name = tensor("new_scale_5")]; + tensor var_699 = sqrt(x = new_scale_5)[name = tensor("op_699")]; + tensor var_700 = real_div(x = new_kv_unnorm_5, y = var_699)[name = tensor("op_700")]; + tensor var_701_perm_0 = const()[name = tensor("op_701_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_807 = transpose(perm = var_807_perm_0, x = out_13)[name = tensor("transpose_43")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_807)[name = tensor("out_15")]; - tensor var_811 = const()[name = tensor("op_811"), val = tensor([1, 2, 256])]; - tensor out_17 = reshape(shape = var_811, x = out_15)[name = tensor("out_17")]; - tensor var_813 = silu(x = input_117)[name = tensor("op_813")]; - tensor input_119 = mul(x = var_813, y = out_17)[name = tensor("input_119")]; - tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_119)[name = tensor("linear_25")]; - tensor x_19 = add(x = input_113, y = ret_out_5)[name = tensor("x_19")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor var_701 = transpose(perm = var_701_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_701)[name = tensor("out_15")]; + tensor var_705 = const()[name = tensor("op_705"), val = tensor([1, 2, 256])]; + tensor out_17 = reshape(shape = var_705, x = out_15)[name = tensor("out_17")]; + tensor var_707 = silu(x = input_97)[name = tensor("op_707")]; + tensor input_99 = mul(x = var_707, y = out_17)[name = tensor("input_99")]; + tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; + tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; + tensor window_13_begin_0 = const()[name = tensor("window_13_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_13_end_0 = const()[name = tensor("window_13_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_13_end_mask_0 = const()[name = tensor("window_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_13_squeeze_mask_0 = const()[name = tensor("window_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_13 = slice_by_index(begin = window_13_begin_0, end = window_13_end_0, end_mask = window_13_end_mask_0, squeeze_mask = window_13_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_13")]; tensor filled_9_begin_0 = const()[name = tensor("filled_9_begin_0"), val = tensor([2, 0])]; tensor filled_9_end_0 = const()[name = tensor("filled_9_end_0"), val = tensor([3, 1])]; tensor filled_9_end_mask_0 = const()[name = tensor("filled_9_end_mask_0"), val = tensor([false, true])]; tensor filled_9_squeeze_mask_0 = const()[name = tensor("filled_9_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_9 = slice_by_index(begin = filled_9_begin_0, end = filled_9_end_0, end_mask = filled_9_end_mask_0, squeeze_mask = filled_9_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_9")]; - tensor var_822_begin_0 = const()[name = tensor("op_822_begin_0"), val = tensor([0, 0, 0])]; - tensor var_822_end_0 = const()[name = tensor("op_822_end_0"), val = tensor([1, 1, 256])]; - tensor var_822_end_mask_0 = const()[name = tensor("op_822_end_mask_0"), val = tensor([true, false, true])]; - tensor var_822 = slice_by_index(begin = var_822_begin_0, end = var_822_end_0, end_mask = var_822_end_mask_0, x = x_19)[name = tensor("op_822")]; - tensor var_825_begin_0 = const()[name = tensor("op_825_begin_0"), val = tensor([0, 1, 0])]; - tensor var_825_end_0 = const()[name = tensor("op_825_end_0"), val = tensor([1, 16, 256])]; - tensor var_825_end_mask_0 = const()[name = tensor("op_825_end_mask_0"), val = tensor([true, true, true])]; - tensor var_825 = slice_by_index(begin = var_825_begin_0, end = var_825_end_0, end_mask = var_825_end_mask_0, x = window_5)[name = tensor("op_825")]; + tensor var_716_begin_0 = const()[name = tensor("op_716_begin_0"), val = tensor([0, 0, 0])]; + tensor var_716_end_0 = const()[name = tensor("op_716_end_0"), val = tensor([1, 1, 256])]; + tensor var_716_end_mask_0 = const()[name = tensor("op_716_end_mask_0"), val = tensor([true, false, true])]; + tensor var_716 = slice_by_index(begin = var_716_begin_0, end = var_716_end_0, end_mask = var_716_end_mask_0, x = x_15)[name = tensor("op_716")]; + tensor var_719_begin_0 = const()[name = tensor("op_719_begin_0"), val = tensor([0, 1, 0])]; + tensor var_719_end_0 = const()[name = tensor("op_719_end_0"), val = tensor([1, 16, 256])]; + tensor var_719_end_mask_0 = const()[name = tensor("op_719_end_mask_0"), val = tensor([true, true, true])]; + tensor var_719 = slice_by_index(begin = var_719_begin_0, end = var_719_end_0, end_mask = var_719_end_mask_0, x = window_13)[name = tensor("op_719")]; tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; - tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_825, var_822))[name = tensor("rolled_9")]; + tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_719, var_716))[name = tensor("rolled_9")]; tensor clip_8 = clip(alpha = var_21, beta = var_20, x = filled_9)[name = tensor("clip_8")]; - tensor var_830 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_830")]; - tensor var_831_dtype_0 = const()[name = tensor("op_831_dtype_0"), val = tensor("fp32")]; - tensor var_832 = const()[name = tensor("op_832"), val = tensor([1, 16, 1])]; - tensor var_831 = cast(dtype = var_831_dtype_0, x = var_830)[name = tensor("cast_135")]; - tensor fill_mask_9 = reshape(shape = var_832, x = var_831)[name = tensor("fill_mask_9")]; + tensor var_724 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_724")]; + tensor var_725_dtype_0 = const()[name = tensor("op_725_dtype_0"), val = tensor("fp32")]; + tensor var_726 = const()[name = tensor("op_726"), val = tensor([1, 16, 1])]; + tensor var_725 = cast(dtype = var_725_dtype_0, x = var_724)[name = tensor("cast_135")]; + tensor fill_mask_9 = reshape(shape = var_726, x = var_725)[name = tensor("fill_mask_9")]; tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_822)[name = tensor("x_expanded_9")]; - tensor var_836 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_836")]; - tensor var_837 = mul(x = window_5, y = var_836)[name = tensor("op_837")]; - tensor var_838 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_838")]; - tensor filled_window_9 = add(x = var_837, y = var_838)[name = tensor("filled_window_9")]; - tensor var_840 = greater_equal(x = filled_9, y = var_16)[name = tensor("op_840")]; - tensor var_841_dtype_0 = const()[name = tensor("op_841_dtype_0"), val = tensor("fp32")]; - tensor var_842 = const()[name = tensor("op_842"), val = tensor([1, 1, 1])]; - tensor var_841 = cast(dtype = var_841_dtype_0, x = var_840)[name = tensor("cast_134")]; - tensor use_roll_9 = reshape(shape = var_842, x = var_841)[name = tensor("use_roll_9")]; - tensor var_844 = sub(x = var_17, y = use_roll_9)[name = tensor("op_844")]; - tensor var_845 = mul(x = filled_window_9, y = var_844)[name = tensor("op_845")]; - tensor var_846 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_846")]; - tensor input_121 = add(x = var_845, y = var_846)[name = tensor("input_121")]; - tensor var_848 = const()[name = tensor("op_848"), val = tensor(0x1p+0)]; - tensor var_849 = add(x = filled_9, y = var_848)[name = tensor("op_849")]; - tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_849)[name = tensor("clip_9")]; - tensor x_21_axes_0 = const()[name = tensor("x_21_axes_0"), val = tensor([-1])]; - tensor x_21 = layer_norm(axes = x_21_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_121)[name = tensor("x_21")]; - tensor input_123_perm_0 = const()[name = tensor("input_123_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_35_pad_type_0 = const()[name = tensor("inputs_35_pad_type_0"), val = tensor("valid")]; - tensor inputs_35_strides_0 = const()[name = tensor("inputs_35_strides_0"), val = tensor([1])]; - tensor inputs_35_pad_0 = const()[name = tensor("inputs_35_pad_0"), val = tensor([0, 0])]; - tensor inputs_35_dilations_0 = const()[name = tensor("inputs_35_dilations_0"), val = tensor([1])]; - tensor inputs_35_groups_0 = const()[name = tensor("inputs_35_groups_0"), val = tensor(1)]; - tensor input_123 = transpose(perm = input_123_perm_0, x = x_21)[name = tensor("transpose_42")]; - tensor inputs_35 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_35_dilations_0, groups = inputs_35_groups_0, pad = inputs_35_pad_0, pad_type = inputs_35_pad_type_0, strides = inputs_35_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_123)[name = tensor("inputs_35")]; - tensor var_870_split_sizes_0 = const()[name = tensor("op_870_split_sizes_0"), val = tensor([256, 256])]; - tensor var_870_axis_0 = const()[name = tensor("op_870_axis_0"), val = tensor(1)]; - tensor var_870_0, tensor var_870_1 = split(axis = var_870_axis_0, split_sizes = var_870_split_sizes_0, x = inputs_35)[name = tensor("op_870")]; - tensor var_872 = sigmoid(x = var_870_1)[name = tensor("op_872")]; - tensor inputs_37 = mul(x = var_870_0, y = var_872)[name = tensor("inputs_37")]; - tensor outputs_aug_9_pad_type_0 = const()[name = tensor("outputs_aug_9_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_9_pad_0 = const()[name = tensor("outputs_aug_9_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_9_groups_0 = const()[name = tensor("outputs_aug_9_groups_0"), val = tensor(256)]; - tensor outputs_aug_9_strides_0 = const()[name = tensor("outputs_aug_9_strides_0"), val = tensor([1])]; - tensor outputs_aug_9_dilations_0 = const()[name = tensor("outputs_aug_9_dilations_0"), val = tensor([1])]; - tensor outputs_aug_9 = conv(dilations = outputs_aug_9_dilations_0, groups = outputs_aug_9_groups_0, pad = outputs_aug_9_pad_0, pad_type = outputs_aug_9_pad_type_0, strides = outputs_aug_9_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_37)[name = tensor("outputs_aug_9")]; - tensor input_125_begin_0 = const()[name = tensor("input_125_begin_0"), val = tensor([0, 0, 0])]; - tensor input_125_end_0 = const()[name = tensor("input_125_end_0"), val = tensor([1, 256, 16])]; - tensor input_125_end_mask_0 = const()[name = tensor("input_125_end_mask_0"), val = tensor([true, true, false])]; - tensor input_125 = slice_by_index(begin = input_125_begin_0, end = input_125_end_0, end_mask = input_125_end_mask_0, x = outputs_aug_9)[name = tensor("input_125")]; - tensor inputs_39 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_125)[name = tensor("inputs_39")]; - tensor input_127 = silu(x = inputs_39)[name = tensor("input_127")]; - tensor input_129_pad_type_0 = const()[name = tensor("input_129_pad_type_0"), val = tensor("valid")]; - tensor input_129_strides_0 = const()[name = tensor("input_129_strides_0"), val = tensor([1])]; - tensor input_129_pad_0 = const()[name = tensor("input_129_pad_0"), val = tensor([0, 0])]; - tensor input_129_dilations_0 = const()[name = tensor("input_129_dilations_0"), val = tensor([1])]; - tensor input_129_groups_0 = const()[name = tensor("input_129_groups_0"), val = tensor(1)]; - tensor input_129 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_129_dilations_0, groups = input_129_groups_0, pad = input_129_pad_0, pad_type = input_129_pad_type_0, strides = input_129_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_127)[name = tensor("input_129")]; - tensor conv_out_9_perm_0 = const()[name = tensor("conv_out_9_perm_0"), val = tensor([0, 2, 1])]; - tensor var_903_begin_0 = const()[name = tensor("op_903_begin_0"), val = tensor([0, -1, 0])]; - tensor var_903_end_0 = const()[name = tensor("op_903_end_0"), val = tensor([1, 16, 256])]; - tensor var_903_end_mask_0 = const()[name = tensor("op_903_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_9 = transpose(perm = conv_out_9_perm_0, x = input_129)[name = tensor("transpose_41")]; - tensor var_903 = slice_by_index(begin = var_903_begin_0, end = var_903_end_0, end_mask = var_903_end_mask_0, x = conv_out_9)[name = tensor("op_903")]; - tensor var_906_begin_0 = const()[name = tensor("op_906_begin_0"), val = tensor([0, 1, 0])]; - tensor var_906_end_0 = const()[name = tensor("op_906_end_0"), val = tensor([1, 1, 256])]; - tensor var_906_end_mask_0 = const()[name = tensor("op_906_end_mask_0"), val = tensor([true, true, true])]; - tensor var_906 = slice_by_index(begin = var_906_begin_0, end = var_906_end_0, end_mask = var_906_end_mask_0, x = x_19)[name = tensor("op_906")]; - tensor var_909_begin_0 = const()[name = tensor("op_909_begin_0"), val = tensor([0, 1, 0])]; - tensor var_909_end_0 = const()[name = tensor("op_909_end_0"), val = tensor([1, 16, 256])]; - tensor var_909_end_mask_0 = const()[name = tensor("op_909_end_mask_0"), val = tensor([true, true, true])]; - tensor var_909 = slice_by_index(begin = var_909_begin_0, end = var_909_end_0, end_mask = var_909_end_mask_0, x = input_121)[name = tensor("op_909")]; + tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_716)[name = tensor("x_expanded_9")]; + tensor var_730 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_730")]; + tensor var_731 = mul(x = window_13, y = var_730)[name = tensor("op_731")]; + tensor var_732 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_732")]; + tensor filled_window_9 = add(x = var_731, y = var_732)[name = tensor("filled_window_9")]; + tensor var_734 = greater_equal(x = filled_9, y = var_16)[name = tensor("op_734")]; + tensor var_735_dtype_0 = const()[name = tensor("op_735_dtype_0"), val = tensor("fp32")]; + tensor var_736 = const()[name = tensor("op_736"), val = tensor([1, 1, 1])]; + tensor var_735 = cast(dtype = var_735_dtype_0, x = var_734)[name = tensor("cast_134")]; + tensor use_roll_9 = reshape(shape = var_736, x = var_735)[name = tensor("use_roll_9")]; + tensor var_738 = sub(x = var_17, y = use_roll_9)[name = tensor("op_738")]; + tensor var_739 = mul(x = filled_window_9, y = var_738)[name = tensor("op_739")]; + tensor var_740 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_740")]; + tensor window_15 = add(x = var_739, y = var_740)[name = tensor("window_15")]; + tensor var_742 = const()[name = tensor("op_742"), val = tensor(0x1p+0)]; + tensor var_743 = add(x = filled_9, y = var_742)[name = tensor("op_743")]; + tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_743)[name = tensor("clip_9")]; + tensor var_746_begin_0 = const()[name = tensor("op_746_begin_0"), val = tensor([0, 1, 0])]; + tensor var_746_end_0 = const()[name = tensor("op_746_end_0"), val = tensor([1, 1, 256])]; + tensor var_746_end_mask_0 = const()[name = tensor("op_746_end_mask_0"), val = tensor([true, true, true])]; + tensor var_746 = slice_by_index(begin = var_746_begin_0, end = var_746_end_0, end_mask = var_746_end_mask_0, x = x_15)[name = tensor("op_746")]; + tensor var_749_begin_0 = const()[name = tensor("op_749_begin_0"), val = tensor([0, 1, 0])]; + tensor var_749_end_0 = const()[name = tensor("op_749_end_0"), val = tensor([1, 16, 256])]; + tensor var_749_end_mask_0 = const()[name = tensor("op_749_end_mask_0"), val = tensor([true, true, true])]; + tensor var_749 = slice_by_index(begin = var_749_begin_0, end = var_749_end_0, end_mask = var_749_end_mask_0, x = window_15)[name = tensor("op_749")]; tensor rolled_11_interleave_0 = const()[name = tensor("rolled_11_interleave_0"), val = tensor(false)]; - tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_909, var_906))[name = tensor("rolled_11")]; + tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_749, var_746))[name = tensor("rolled_11")]; tensor clip_10 = clip(alpha = var_21, beta = var_20, x = clip_9)[name = tensor("clip_10")]; - tensor var_914 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_914")]; - tensor var_915_dtype_0 = const()[name = tensor("op_915_dtype_0"), val = tensor("fp32")]; - tensor var_916 = const()[name = tensor("op_916"), val = tensor([1, 16, 1])]; - tensor var_915 = cast(dtype = var_915_dtype_0, x = var_914)[name = tensor("cast_133")]; - tensor fill_mask_11 = reshape(shape = var_916, x = var_915)[name = tensor("fill_mask_11")]; + tensor var_754 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_754")]; + tensor var_755_dtype_0 = const()[name = tensor("op_755_dtype_0"), val = tensor("fp32")]; + tensor var_756 = const()[name = tensor("op_756"), val = tensor([1, 16, 1])]; + tensor var_755 = cast(dtype = var_755_dtype_0, x = var_754)[name = tensor("cast_133")]; + tensor fill_mask_11 = reshape(shape = var_756, x = var_755)[name = tensor("fill_mask_11")]; tensor x_expanded_11_reps_0 = const()[name = tensor("x_expanded_11_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_906)[name = tensor("x_expanded_11")]; - tensor var_920 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_920")]; - tensor var_921 = mul(x = input_121, y = var_920)[name = tensor("op_921")]; - tensor var_922 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_922")]; - tensor filled_window_11 = add(x = var_921, y = var_922)[name = tensor("filled_window_11")]; - tensor var_924 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_924")]; - tensor var_925_dtype_0 = const()[name = tensor("op_925_dtype_0"), val = tensor("fp32")]; - tensor var_926 = const()[name = tensor("op_926"), val = tensor([1, 1, 1])]; - tensor var_925 = cast(dtype = var_925_dtype_0, x = var_924)[name = tensor("cast_132")]; - tensor use_roll_11 = reshape(shape = var_926, x = var_925)[name = tensor("use_roll_11")]; - tensor var_928 = sub(x = var_17, y = use_roll_11)[name = tensor("op_928")]; - tensor var_929 = mul(x = filled_window_11, y = var_928)[name = tensor("op_929")]; - tensor var_930 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_930")]; - tensor input_131 = add(x = var_929, y = var_930)[name = tensor("input_131")]; - tensor var_932 = const()[name = tensor("op_932"), val = tensor(0x1p+0)]; - tensor var_933 = add(x = clip_9, y = var_932)[name = tensor("op_933")]; - tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_933)[name = tensor("clip_11")]; - tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; - tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_131)[name = tensor("x_23")]; - tensor input_133_perm_0 = const()[name = tensor("input_133_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_41_pad_type_0 = const()[name = tensor("inputs_41_pad_type_0"), val = tensor("valid")]; - tensor inputs_41_strides_0 = const()[name = tensor("inputs_41_strides_0"), val = tensor([1])]; - tensor inputs_41_pad_0 = const()[name = tensor("inputs_41_pad_0"), val = tensor([0, 0])]; - tensor inputs_41_dilations_0 = const()[name = tensor("inputs_41_dilations_0"), val = tensor([1])]; - tensor inputs_41_groups_0 = const()[name = tensor("inputs_41_groups_0"), val = tensor(1)]; - tensor input_133 = transpose(perm = input_133_perm_0, x = x_23)[name = tensor("transpose_40")]; - tensor inputs_41 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_41_dilations_0, groups = inputs_41_groups_0, pad = inputs_41_pad_0, pad_type = inputs_41_pad_type_0, strides = inputs_41_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_133)[name = tensor("inputs_41")]; - tensor var_954_split_sizes_0 = const()[name = tensor("op_954_split_sizes_0"), val = tensor([256, 256])]; - tensor var_954_axis_0 = const()[name = tensor("op_954_axis_0"), val = tensor(1)]; - tensor var_954_0, tensor var_954_1 = split(axis = var_954_axis_0, split_sizes = var_954_split_sizes_0, x = inputs_41)[name = tensor("op_954")]; - tensor var_956 = sigmoid(x = var_954_1)[name = tensor("op_956")]; - tensor inputs_43 = mul(x = var_954_0, y = var_956)[name = tensor("inputs_43")]; - tensor outputs_aug_11_pad_type_0 = const()[name = tensor("outputs_aug_11_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_11_pad_0 = const()[name = tensor("outputs_aug_11_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_11_groups_0 = const()[name = tensor("outputs_aug_11_groups_0"), val = tensor(256)]; - tensor outputs_aug_11_strides_0 = const()[name = tensor("outputs_aug_11_strides_0"), val = tensor([1])]; - tensor outputs_aug_11_dilations_0 = const()[name = tensor("outputs_aug_11_dilations_0"), val = tensor([1])]; - tensor outputs_aug_11 = conv(dilations = outputs_aug_11_dilations_0, groups = outputs_aug_11_groups_0, pad = outputs_aug_11_pad_0, pad_type = outputs_aug_11_pad_type_0, strides = outputs_aug_11_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_43)[name = tensor("outputs_aug_11")]; - tensor input_135_begin_0 = const()[name = tensor("input_135_begin_0"), val = tensor([0, 0, 0])]; - tensor input_135_end_0 = const()[name = tensor("input_135_end_0"), val = tensor([1, 256, 16])]; - tensor input_135_end_mask_0 = const()[name = tensor("input_135_end_mask_0"), val = tensor([true, true, false])]; - tensor input_135 = slice_by_index(begin = input_135_begin_0, end = input_135_end_0, end_mask = input_135_end_mask_0, x = outputs_aug_11)[name = tensor("input_135")]; - tensor inputs_45 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_135)[name = tensor("inputs_45")]; - tensor input_137 = silu(x = inputs_45)[name = tensor("input_137")]; - tensor input_139_pad_type_0 = const()[name = tensor("input_139_pad_type_0"), val = tensor("valid")]; - tensor input_139_strides_0 = const()[name = tensor("input_139_strides_0"), val = tensor([1])]; - tensor input_139_pad_0 = const()[name = tensor("input_139_pad_0"), val = tensor([0, 0])]; - tensor input_139_dilations_0 = const()[name = tensor("input_139_dilations_0"), val = tensor([1])]; - tensor input_139_groups_0 = const()[name = tensor("input_139_groups_0"), val = tensor(1)]; - tensor input_139 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_139_dilations_0, groups = input_139_groups_0, pad = input_139_pad_0, pad_type = input_139_pad_type_0, strides = input_139_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_137)[name = tensor("input_139")]; - tensor conv_out_11_perm_0 = const()[name = tensor("conv_out_11_perm_0"), val = tensor([0, 2, 1])]; - tensor var_987_begin_0 = const()[name = tensor("op_987_begin_0"), val = tensor([0, -1, 0])]; - tensor var_987_end_0 = const()[name = tensor("op_987_end_0"), val = tensor([1, 16, 256])]; - tensor var_987_end_mask_0 = const()[name = tensor("op_987_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_11 = transpose(perm = conv_out_11_perm_0, x = input_139)[name = tensor("transpose_39")]; - tensor var_987 = slice_by_index(begin = var_987_begin_0, end = var_987_end_0, end_mask = var_987_end_mask_0, x = conv_out_11)[name = tensor("op_987")]; - tensor var_990_interleave_0 = const()[name = tensor("op_990_interleave_0"), val = tensor(false)]; - tensor var_990 = concat(axis = var_33, interleave = var_990_interleave_0, values = (var_903, var_987))[name = tensor("op_990")]; - tensor input_141 = add(x = x_19, y = var_990)[name = tensor("input_141")]; - tensor input_143_axes_0 = const()[name = tensor("input_143_axes_0"), val = tensor([-1])]; - tensor input_143 = layer_norm(axes = input_143_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_141)[name = tensor("input_143")]; - tensor inputs_47 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_143)[name = tensor("linear_26")]; - tensor input_145 = silu(x = inputs_47)[name = tensor("input_145")]; - tensor input_149 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_145)[name = tensor("linear_27")]; - tensor var_1013 = const()[name = tensor("op_1013"), val = tensor(0x1p-1)]; - tensor var_1014 = mul(x = input_149, y = var_1013)[name = tensor("op_1014")]; - tensor input_151 = add(x = var_1014, y = input_141)[name = tensor("input_151")]; - tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; - tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_151)[name = tensor("input_153")]; - tensor input_155_axes_0 = const()[name = tensor("input_155_axes_0"), val = tensor([-1])]; - tensor input_155 = layer_norm(axes = input_155_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_153)[name = tensor("input_155")]; - tensor inputs_49 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_155)[name = tensor("linear_28")]; - tensor input_157 = silu(x = inputs_49)[name = tensor("input_157")]; - tensor input_161 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_157)[name = tensor("linear_29")]; - tensor var_1043 = const()[name = tensor("op_1043"), val = tensor(0x1p-1)]; - tensor var_1044 = mul(x = input_161, y = var_1043)[name = tensor("op_1044")]; - tensor input_163 = add(x = var_1044, y = input_153)[name = tensor("input_163")]; - tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; - tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_163)[name = tensor("x_25")]; + tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_746)[name = tensor("x_expanded_11")]; + tensor var_760 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_760")]; + tensor var_761 = mul(x = window_15, y = var_760)[name = tensor("op_761")]; + tensor var_762 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_762")]; + tensor filled_window_11 = add(x = var_761, y = var_762)[name = tensor("filled_window_11")]; + tensor var_764 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_764")]; + tensor var_765_dtype_0 = const()[name = tensor("op_765_dtype_0"), val = tensor("fp32")]; + tensor var_766 = const()[name = tensor("op_766"), val = tensor([1, 1, 1])]; + tensor var_765 = cast(dtype = var_765_dtype_0, x = var_764)[name = tensor("cast_132")]; + tensor use_roll_11 = reshape(shape = var_766, x = var_765)[name = tensor("use_roll_11")]; + tensor var_768 = sub(x = var_17, y = use_roll_11)[name = tensor("op_768")]; + tensor var_769 = mul(x = filled_window_11, y = var_768)[name = tensor("op_769")]; + tensor var_770 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_770")]; + tensor window_17 = add(x = var_769, y = var_770)[name = tensor("window_17")]; + tensor var_772 = const()[name = tensor("op_772"), val = tensor(0x1p+0)]; + tensor var_773 = add(x = clip_9, y = var_772)[name = tensor("op_773")]; + tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_773)[name = tensor("clip_11")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_30, interleave = input_101_interleave_0, values = (window_15, window_17))[name = tensor("input_101")]; + tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; + tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; + tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_23_pad_type_0 = const()[name = tensor("inputs_23_pad_type_0"), val = tensor("valid")]; + tensor inputs_23_strides_0 = const()[name = tensor("inputs_23_strides_0"), val = tensor([1])]; + tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; + tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; + tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; + tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; + tensor var_796_split_sizes_0 = const()[name = tensor("op_796_split_sizes_0"), val = tensor([256, 256])]; + tensor var_796_axis_0 = const()[name = tensor("op_796_axis_0"), val = tensor(1)]; + tensor var_796_0, tensor var_796_1 = split(axis = var_796_axis_0, split_sizes = var_796_split_sizes_0, x = inputs_23)[name = tensor("op_796")]; + tensor var_798 = sigmoid(x = var_796_1)[name = tensor("op_798")]; + tensor inputs_25 = mul(x = var_796_0, y = var_798)[name = tensor("inputs_25")]; + tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; + tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; + tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; + tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_25)[name = tensor("outputs_aug_5")]; + tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; + tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([2, 256, 16])]; + tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; + tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_5)[name = tensor("input_105")]; + tensor inputs_27 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_105)[name = tensor("inputs_27")]; + tensor input_107 = silu(x = inputs_27)[name = tensor("input_107")]; + tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; + tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; + tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; + tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; + tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; + tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; + tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; + tensor var_829_begin_0 = const()[name = tensor("op_829_begin_0"), val = tensor([0, -1, 0])]; + tensor var_829_end_0 = const()[name = tensor("op_829_end_0"), val = tensor([2, 16, 256])]; + tensor var_829_end_mask_0 = const()[name = tensor("op_829_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_829 = slice_by_index(begin = var_829_begin_0, end = var_829_end_0, end_mask = var_829_end_mask_0, x = conv_out_5)[name = tensor("op_829")]; + tensor var_831_perm_0 = const()[name = tensor("op_831_perm_0"), val = tensor([1, 0, 2])]; + tensor var_831 = transpose(perm = var_831_perm_0, x = var_829)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_831)[name = tensor("input_111")]; + tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; + tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; + tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; + tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; + tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; + tensor var_854 = const()[name = tensor("op_854"), val = tensor(0x1p-1)]; + tensor var_855 = mul(x = input_119, y = var_854)[name = tensor("op_855")]; + tensor input_121 = add(x = var_855, y = input_111)[name = tensor("input_121")]; + tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; + tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; + tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; + tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; + tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; + tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; + tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; + tensor var_884 = const()[name = tensor("op_884"), val = tensor(0x1p-1)]; + tensor var_885 = mul(x = input_131, y = var_884)[name = tensor("op_885")]; + tensor input_133 = add(x = var_885, y = input_123)[name = tensor("input_133")]; + tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; + tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; tensor prev_kv_7_end_0 = const()[name = tensor("prev_kv_7_end_0"), val = tensor([4, 1, 4, 64, 64])]; tensor prev_kv_7_end_mask_0 = const()[name = tensor("prev_kv_7_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -984,236 +874,199 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_1058 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_25)[name = tensor("linear_30")]; - tensor var_1059 = const()[name = tensor("op_1059"), val = tensor([1, 2, 4, 64])]; - tensor var_1060 = reshape(shape = var_1059, x = var_1058)[name = tensor("op_1060")]; + tensor var_899 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_900 = const()[name = tensor("op_900"), val = tensor([1, 2, 4, 64])]; + tensor var_901 = reshape(shape = var_900, x = var_899)[name = tensor("op_901")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1064 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_25)[name = tensor("linear_31")]; - tensor var_1065 = const()[name = tensor("op_1065"), val = tensor(0x1p-3)]; - tensor var_1066 = mul(x = var_1064, y = var_1065)[name = tensor("op_1066")]; - tensor var_1067 = const()[name = tensor("op_1067"), val = tensor([1, 2, 4, 64])]; - tensor var_1068 = reshape(shape = var_1067, x = var_1066)[name = tensor("op_1068")]; + tensor var_905 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_906 = const()[name = tensor("op_906"), val = tensor(0x1p-3)]; + tensor var_907 = mul(x = var_905, y = var_906)[name = tensor("op_907")]; + tensor var_908 = const()[name = tensor("op_908"), val = tensor([1, 2, 4, 64])]; + tensor var_909 = reshape(shape = var_908, x = var_907)[name = tensor("op_909")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1072 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_25)[name = tensor("linear_32")]; - tensor var_1073 = const()[name = tensor("op_1073"), val = tensor([1, 2, 4, 64])]; - tensor var_1074 = reshape(shape = var_1073, x = var_1072)[name = tensor("op_1074")]; + tensor var_913 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_914 = const()[name = tensor("op_914"), val = tensor([1, 2, 4, 64])]; + tensor var_915 = reshape(shape = var_914, x = var_913)[name = tensor("op_915")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_167 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_25)[name = tensor("linear_33")]; + tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; tensor s_t_7 = add(x = prev_scale_7, y = encoder__t_index)[name = tensor("s_t_7")]; tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_1068)[name = tensor("transpose_37")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_1060)[name = tensor("transpose_38")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_909)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_901)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_1084 = const()[name = tensor("op_1084"), val = tensor([2, 1])]; - tensor var_1085 = reshape(shape = var_1084, x = sqrt_s_t_7)[name = tensor("op_1085")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_1085)[name = tensor("M_7")]; - tensor var_1087 = mul(x = qk_7, y = M_7)[name = tensor("op_1087")]; + tensor var_925 = const()[name = tensor("op_925"), val = tensor([2, 1])]; + tensor var_926 = reshape(shape = var_925, x = sqrt_s_t_7)[name = tensor("op_926")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_926)[name = tensor("M_7")]; + tensor var_928 = mul(x = qk_7, y = M_7)[name = tensor("op_928")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_1074)[name = tensor("transpose_36")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1087, y = v_7)[name = tensor("inner_7")]; - tensor var_1089_transpose_x_0 = const()[name = tensor("op_1089_transpose_x_0"), val = tensor(false)]; - tensor var_1089_transpose_y_0 = const()[name = tensor("op_1089_transpose_y_0"), val = tensor(false)]; - tensor var_1089 = matmul(transpose_x = var_1089_transpose_x_0, transpose_y = var_1089_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1089")]; - tensor var_1090 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1090")]; - tensor var_1091 = const()[name = tensor("op_1091"), val = tensor([1, 1, 2, 1])]; - tensor var_1092 = reshape(shape = var_1091, x = var_1090)[name = tensor("op_1092")]; - tensor cross_7 = mul(x = var_1089, y = var_1092)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_915)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_928, y = v_7)[name = tensor("inner_7")]; + tensor var_930_transpose_x_0 = const()[name = tensor("op_930_transpose_x_0"), val = tensor(false)]; + tensor var_930_transpose_y_0 = const()[name = tensor("op_930_transpose_y_0"), val = tensor(false)]; + tensor var_930 = matmul(transpose_x = var_930_transpose_x_0, transpose_y = var_930_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_930")]; + tensor var_931 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_931")]; + tensor var_932 = const()[name = tensor("op_932"), val = tensor([1, 1, 2, 1])]; + tensor var_933 = reshape(shape = var_932, x = var_931)[name = tensor("op_933")]; + tensor cross_7 = mul(x = var_930, y = var_933)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_1095 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1095")]; - tensor var_1097_transpose_x_1 = const()[name = tensor("op_1097_transpose_x_1"), val = tensor(true)]; - tensor var_1097_transpose_y_1 = const()[name = tensor("op_1097_transpose_y_1"), val = tensor(false)]; - tensor var_1097 = matmul(transpose_x = var_1097_transpose_x_1, transpose_y = var_1097_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1097")]; - tensor new_kv_unnorm_7 = add(x = var_1095, y = var_1097)[name = tensor("new_kv_unnorm_7")]; - tensor var_1099 = const()[name = tensor("op_1099"), val = tensor(0x1p+1)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_1099)[name = tensor("new_scale_7")]; - tensor var_1101 = sqrt(x = new_scale_7)[name = tensor("op_1101")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1101)[name = tensor("nkv_1")]; - tensor var_1103_perm_0 = const()[name = tensor("op_1103_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_936 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_936")]; + tensor var_938_transpose_x_1 = const()[name = tensor("op_938_transpose_x_1"), val = tensor(true)]; + tensor var_938_transpose_y_1 = const()[name = tensor("op_938_transpose_y_1"), val = tensor(false)]; + tensor var_938 = matmul(transpose_x = var_938_transpose_x_1, transpose_y = var_938_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_938")]; + tensor new_kv_unnorm_7 = add(x = var_936, y = var_938)[name = tensor("new_kv_unnorm_7")]; + tensor var_940 = const()[name = tensor("op_940"), val = tensor(0x1p+1)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_940)[name = tensor("new_scale_7")]; + tensor var_942 = sqrt(x = new_scale_7)[name = tensor("op_942")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_942)[name = tensor("nkv_1")]; + tensor var_944_perm_0 = const()[name = tensor("op_944_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_1103 = transpose(perm = var_1103_perm_0, x = out_19)[name = tensor("transpose_35")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1103)[name = tensor("out_21")]; - tensor var_1107 = const()[name = tensor("op_1107"), val = tensor([1, 2, 256])]; - tensor out_23 = reshape(shape = var_1107, x = out_21)[name = tensor("out_23")]; - tensor var_1109 = silu(x = input_167)[name = tensor("op_1109")]; - tensor input_169 = mul(x = var_1109, y = out_23)[name = tensor("input_169")]; - tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_169)[name = tensor("linear_34")]; - tensor x_27 = add(x = input_163, y = ret_out_7)[name = tensor("x_27")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor var_944 = transpose(perm = var_944_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_944)[name = tensor("out_21")]; + tensor var_948 = const()[name = tensor("op_948"), val = tensor([1, 2, 256])]; + tensor out_23 = reshape(shape = var_948, x = out_21)[name = tensor("out_23")]; + tensor var_950 = silu(x = input_137)[name = tensor("op_950")]; + tensor input_139 = mul(x = var_950, y = out_23)[name = tensor("input_139")]; + tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; + tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; + tensor window_19_begin_0 = const()[name = tensor("window_19_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_19_end_0 = const()[name = tensor("window_19_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_19_end_mask_0 = const()[name = tensor("window_19_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_19_squeeze_mask_0 = const()[name = tensor("window_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_19 = slice_by_index(begin = window_19_begin_0, end = window_19_end_0, end_mask = window_19_end_mask_0, squeeze_mask = window_19_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_19")]; tensor filled_13_begin_0 = const()[name = tensor("filled_13_begin_0"), val = tensor([3, 0])]; tensor filled_13_end_0 = const()[name = tensor("filled_13_end_0"), val = tensor([4, 1])]; tensor filled_13_end_mask_0 = const()[name = tensor("filled_13_end_mask_0"), val = tensor([false, true])]; tensor filled_13_squeeze_mask_0 = const()[name = tensor("filled_13_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_13 = slice_by_index(begin = filled_13_begin_0, end = filled_13_end_0, end_mask = filled_13_end_mask_0, squeeze_mask = filled_13_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_13")]; - tensor var_1118_begin_0 = const()[name = tensor("op_1118_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1118_end_0 = const()[name = tensor("op_1118_end_0"), val = tensor([1, 1, 256])]; - tensor var_1118_end_mask_0 = const()[name = tensor("op_1118_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1118 = slice_by_index(begin = var_1118_begin_0, end = var_1118_end_0, end_mask = var_1118_end_mask_0, x = x_27)[name = tensor("op_1118")]; - tensor var_1121_begin_0 = const()[name = tensor("op_1121_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1121_end_0 = const()[name = tensor("op_1121_end_0"), val = tensor([1, 16, 256])]; - tensor var_1121_end_mask_0 = const()[name = tensor("op_1121_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1121 = slice_by_index(begin = var_1121_begin_0, end = var_1121_end_0, end_mask = var_1121_end_mask_0, x = window)[name = tensor("op_1121")]; + tensor var_959_begin_0 = const()[name = tensor("op_959_begin_0"), val = tensor([0, 0, 0])]; + tensor var_959_end_0 = const()[name = tensor("op_959_end_0"), val = tensor([1, 1, 256])]; + tensor var_959_end_mask_0 = const()[name = tensor("op_959_end_mask_0"), val = tensor([true, false, true])]; + tensor var_959 = slice_by_index(begin = var_959_begin_0, end = var_959_end_0, end_mask = var_959_end_mask_0, x = x_21)[name = tensor("op_959")]; + tensor var_962_begin_0 = const()[name = tensor("op_962_begin_0"), val = tensor([0, 1, 0])]; + tensor var_962_end_0 = const()[name = tensor("op_962_end_0"), val = tensor([1, 16, 256])]; + tensor var_962_end_mask_0 = const()[name = tensor("op_962_end_mask_0"), val = tensor([true, true, true])]; + tensor var_962 = slice_by_index(begin = var_962_begin_0, end = var_962_end_0, end_mask = var_962_end_mask_0, x = window_19)[name = tensor("op_962")]; tensor rolled_13_interleave_0 = const()[name = tensor("rolled_13_interleave_0"), val = tensor(false)]; - tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_1121, var_1118))[name = tensor("rolled_13")]; + tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_962, var_959))[name = tensor("rolled_13")]; tensor clip_12 = clip(alpha = var_21, beta = var_20, x = filled_13)[name = tensor("clip_12")]; - tensor var_1126 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_1126")]; - tensor var_1127_dtype_0 = const()[name = tensor("op_1127_dtype_0"), val = tensor("fp32")]; - tensor var_1128 = const()[name = tensor("op_1128"), val = tensor([1, 16, 1])]; - tensor var_1127 = cast(dtype = var_1127_dtype_0, x = var_1126)[name = tensor("cast_131")]; - tensor fill_mask_13 = reshape(shape = var_1128, x = var_1127)[name = tensor("fill_mask_13")]; + tensor var_967 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_967")]; + tensor var_968_dtype_0 = const()[name = tensor("op_968_dtype_0"), val = tensor("fp32")]; + tensor var_969 = const()[name = tensor("op_969"), val = tensor([1, 16, 1])]; + tensor var_968 = cast(dtype = var_968_dtype_0, x = var_967)[name = tensor("cast_131")]; + tensor fill_mask_13 = reshape(shape = var_969, x = var_968)[name = tensor("fill_mask_13")]; tensor x_expanded_13_reps_0 = const()[name = tensor("x_expanded_13_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_1118)[name = tensor("x_expanded_13")]; - tensor var_1132 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_1132")]; - tensor var_1133 = mul(x = window, y = var_1132)[name = tensor("op_1133")]; - tensor var_1134 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_1134")]; - tensor filled_window_13 = add(x = var_1133, y = var_1134)[name = tensor("filled_window_13")]; - tensor var_1136 = greater_equal(x = filled_13, y = var_16)[name = tensor("op_1136")]; - tensor var_1137_dtype_0 = const()[name = tensor("op_1137_dtype_0"), val = tensor("fp32")]; - tensor var_1138 = const()[name = tensor("op_1138"), val = tensor([1, 1, 1])]; - tensor var_1137 = cast(dtype = var_1137_dtype_0, x = var_1136)[name = tensor("cast_130")]; - tensor use_roll_13 = reshape(shape = var_1138, x = var_1137)[name = tensor("use_roll_13")]; - tensor var_1140 = sub(x = var_17, y = use_roll_13)[name = tensor("op_1140")]; - tensor var_1141 = mul(x = filled_window_13, y = var_1140)[name = tensor("op_1141")]; - tensor var_1142 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_1142")]; - tensor input_171 = add(x = var_1141, y = var_1142)[name = tensor("input_171")]; - tensor var_1144 = const()[name = tensor("op_1144"), val = tensor(0x1p+0)]; - tensor var_1145 = add(x = filled_13, y = var_1144)[name = tensor("op_1145")]; - tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_1145)[name = tensor("clip_13")]; - tensor x_29_axes_0 = const()[name = tensor("x_29_axes_0"), val = tensor([-1])]; - tensor x_29 = layer_norm(axes = x_29_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_171)[name = tensor("x_29")]; - tensor input_173_perm_0 = const()[name = tensor("input_173_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_51_pad_type_0 = const()[name = tensor("inputs_51_pad_type_0"), val = tensor("valid")]; - tensor inputs_51_strides_0 = const()[name = tensor("inputs_51_strides_0"), val = tensor([1])]; - tensor inputs_51_pad_0 = const()[name = tensor("inputs_51_pad_0"), val = tensor([0, 0])]; - tensor inputs_51_dilations_0 = const()[name = tensor("inputs_51_dilations_0"), val = tensor([1])]; - tensor inputs_51_groups_0 = const()[name = tensor("inputs_51_groups_0"), val = tensor(1)]; - tensor input_173 = transpose(perm = input_173_perm_0, x = x_29)[name = tensor("transpose_34")]; - tensor inputs_51 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_51_dilations_0, groups = inputs_51_groups_0, pad = inputs_51_pad_0, pad_type = inputs_51_pad_type_0, strides = inputs_51_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_173)[name = tensor("inputs_51")]; - tensor var_1166_split_sizes_0 = const()[name = tensor("op_1166_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1166_axis_0 = const()[name = tensor("op_1166_axis_0"), val = tensor(1)]; - tensor var_1166_0, tensor var_1166_1 = split(axis = var_1166_axis_0, split_sizes = var_1166_split_sizes_0, x = inputs_51)[name = tensor("op_1166")]; - tensor var_1168 = sigmoid(x = var_1166_1)[name = tensor("op_1168")]; - tensor inputs_53 = mul(x = var_1166_0, y = var_1168)[name = tensor("inputs_53")]; - tensor outputs_aug_13_pad_type_0 = const()[name = tensor("outputs_aug_13_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_13_pad_0 = const()[name = tensor("outputs_aug_13_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_13_groups_0 = const()[name = tensor("outputs_aug_13_groups_0"), val = tensor(256)]; - tensor outputs_aug_13_strides_0 = const()[name = tensor("outputs_aug_13_strides_0"), val = tensor([1])]; - tensor outputs_aug_13_dilations_0 = const()[name = tensor("outputs_aug_13_dilations_0"), val = tensor([1])]; - tensor outputs_aug_13 = conv(dilations = outputs_aug_13_dilations_0, groups = outputs_aug_13_groups_0, pad = outputs_aug_13_pad_0, pad_type = outputs_aug_13_pad_type_0, strides = outputs_aug_13_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_53)[name = tensor("outputs_aug_13")]; - tensor input_175_begin_0 = const()[name = tensor("input_175_begin_0"), val = tensor([0, 0, 0])]; - tensor input_175_end_0 = const()[name = tensor("input_175_end_0"), val = tensor([1, 256, 16])]; - tensor input_175_end_mask_0 = const()[name = tensor("input_175_end_mask_0"), val = tensor([true, true, false])]; - tensor input_175 = slice_by_index(begin = input_175_begin_0, end = input_175_end_0, end_mask = input_175_end_mask_0, x = outputs_aug_13)[name = tensor("input_175")]; - tensor inputs_55 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_175)[name = tensor("inputs_55")]; - tensor input_177 = silu(x = inputs_55)[name = tensor("input_177")]; - tensor input_179_pad_type_0 = const()[name = tensor("input_179_pad_type_0"), val = tensor("valid")]; - tensor input_179_strides_0 = const()[name = tensor("input_179_strides_0"), val = tensor([1])]; - tensor input_179_pad_0 = const()[name = tensor("input_179_pad_0"), val = tensor([0, 0])]; - tensor input_179_dilations_0 = const()[name = tensor("input_179_dilations_0"), val = tensor([1])]; - tensor input_179_groups_0 = const()[name = tensor("input_179_groups_0"), val = tensor(1)]; - tensor input_179 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_179_dilations_0, groups = input_179_groups_0, pad = input_179_pad_0, pad_type = input_179_pad_type_0, strides = input_179_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_177)[name = tensor("input_179")]; - tensor conv_out_13_perm_0 = const()[name = tensor("conv_out_13_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1199_begin_0 = const()[name = tensor("op_1199_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1199_end_0 = const()[name = tensor("op_1199_end_0"), val = tensor([1, 16, 256])]; - tensor var_1199_end_mask_0 = const()[name = tensor("op_1199_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_13 = transpose(perm = conv_out_13_perm_0, x = input_179)[name = tensor("transpose_33")]; - tensor var_1199 = slice_by_index(begin = var_1199_begin_0, end = var_1199_end_0, end_mask = var_1199_end_mask_0, x = conv_out_13)[name = tensor("op_1199")]; - tensor var_1202_begin_0 = const()[name = tensor("op_1202_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1202_end_0 = const()[name = tensor("op_1202_end_0"), val = tensor([1, 1, 256])]; - tensor var_1202_end_mask_0 = const()[name = tensor("op_1202_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1202 = slice_by_index(begin = var_1202_begin_0, end = var_1202_end_0, end_mask = var_1202_end_mask_0, x = x_27)[name = tensor("op_1202")]; - tensor var_1205_begin_0 = const()[name = tensor("op_1205_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1205_end_0 = const()[name = tensor("op_1205_end_0"), val = tensor([1, 16, 256])]; - tensor var_1205_end_mask_0 = const()[name = tensor("op_1205_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1205 = slice_by_index(begin = var_1205_begin_0, end = var_1205_end_0, end_mask = var_1205_end_mask_0, x = input_171)[name = tensor("op_1205")]; + tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_959)[name = tensor("x_expanded_13")]; + tensor var_973 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_973")]; + tensor var_974 = mul(x = window_19, y = var_973)[name = tensor("op_974")]; + tensor var_975 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_975")]; + tensor filled_window_13 = add(x = var_974, y = var_975)[name = tensor("filled_window_13")]; + tensor var_977 = greater_equal(x = filled_13, y = var_16)[name = tensor("op_977")]; + tensor var_978_dtype_0 = const()[name = tensor("op_978_dtype_0"), val = tensor("fp32")]; + tensor var_979 = const()[name = tensor("op_979"), val = tensor([1, 1, 1])]; + tensor var_978 = cast(dtype = var_978_dtype_0, x = var_977)[name = tensor("cast_130")]; + tensor use_roll_13 = reshape(shape = var_979, x = var_978)[name = tensor("use_roll_13")]; + tensor var_981 = sub(x = var_17, y = use_roll_13)[name = tensor("op_981")]; + tensor var_982 = mul(x = filled_window_13, y = var_981)[name = tensor("op_982")]; + tensor var_983 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_983")]; + tensor window_21 = add(x = var_982, y = var_983)[name = tensor("window_21")]; + tensor var_985 = const()[name = tensor("op_985"), val = tensor(0x1p+0)]; + tensor var_986 = add(x = filled_13, y = var_985)[name = tensor("op_986")]; + tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_986)[name = tensor("clip_13")]; + tensor var_989_begin_0 = const()[name = tensor("op_989_begin_0"), val = tensor([0, 1, 0])]; + tensor var_989_end_0 = const()[name = tensor("op_989_end_0"), val = tensor([1, 1, 256])]; + tensor var_989_end_mask_0 = const()[name = tensor("op_989_end_mask_0"), val = tensor([true, true, true])]; + tensor var_989 = slice_by_index(begin = var_989_begin_0, end = var_989_end_0, end_mask = var_989_end_mask_0, x = x_21)[name = tensor("op_989")]; + tensor var_992_begin_0 = const()[name = tensor("op_992_begin_0"), val = tensor([0, 1, 0])]; + tensor var_992_end_0 = const()[name = tensor("op_992_end_0"), val = tensor([1, 16, 256])]; + tensor var_992_end_mask_0 = const()[name = tensor("op_992_end_mask_0"), val = tensor([true, true, true])]; + tensor var_992 = slice_by_index(begin = var_992_begin_0, end = var_992_end_0, end_mask = var_992_end_mask_0, x = window_21)[name = tensor("op_992")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1205, var_1202))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_992, var_989))[name = tensor("rolled")]; tensor clip_14 = clip(alpha = var_21, beta = var_20, x = clip_13)[name = tensor("clip_14")]; - tensor var_1210 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_1210")]; - tensor var_1211_dtype_0 = const()[name = tensor("op_1211_dtype_0"), val = tensor("fp32")]; - tensor var_1212 = const()[name = tensor("op_1212"), val = tensor([1, 16, 1])]; - tensor var_1211 = cast(dtype = var_1211_dtype_0, x = var_1210)[name = tensor("cast_129")]; - tensor fill_mask = reshape(shape = var_1212, x = var_1211)[name = tensor("fill_mask")]; + tensor var_997 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_997")]; + tensor var_998_dtype_0 = const()[name = tensor("op_998_dtype_0"), val = tensor("fp32")]; + tensor var_999 = const()[name = tensor("op_999"), val = tensor([1, 16, 1])]; + tensor var_998 = cast(dtype = var_998_dtype_0, x = var_997)[name = tensor("cast_129")]; + tensor fill_mask = reshape(shape = var_999, x = var_998)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1202)[name = tensor("x_expanded")]; - tensor var_1216 = sub(x = var_17, y = fill_mask)[name = tensor("op_1216")]; - tensor var_1217 = mul(x = input_171, y = var_1216)[name = tensor("op_1217")]; - tensor var_1218 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1218")]; - tensor filled_window = add(x = var_1217, y = var_1218)[name = tensor("filled_window")]; - tensor var_1220 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_1220")]; - tensor var_1221_dtype_0 = const()[name = tensor("op_1221_dtype_0"), val = tensor("fp32")]; - tensor var_1222 = const()[name = tensor("op_1222"), val = tensor([1, 1, 1])]; - tensor var_1221 = cast(dtype = var_1221_dtype_0, x = var_1220)[name = tensor("cast_128")]; - tensor use_roll = reshape(shape = var_1222, x = var_1221)[name = tensor("use_roll")]; - tensor var_1224 = sub(x = var_17, y = use_roll)[name = tensor("op_1224")]; - tensor var_1225 = mul(x = filled_window, y = var_1224)[name = tensor("op_1225")]; - tensor var_1226 = mul(x = rolled, y = use_roll)[name = tensor("op_1226")]; - tensor input_181 = add(x = var_1225, y = var_1226)[name = tensor("input_181")]; - tensor var_1228 = const()[name = tensor("op_1228"), val = tensor(0x1p+0)]; - tensor var_1229 = add(x = clip_13, y = var_1228)[name = tensor("op_1229")]; - tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_1229)[name = tensor("clip_15")]; - tensor x_31_axes_0 = const()[name = tensor("x_31_axes_0"), val = tensor([-1])]; - tensor x_31 = layer_norm(axes = x_31_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_181)[name = tensor("x_31")]; - tensor input_183_perm_0 = const()[name = tensor("input_183_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_57_pad_type_0 = const()[name = tensor("inputs_57_pad_type_0"), val = tensor("valid")]; - tensor inputs_57_strides_0 = const()[name = tensor("inputs_57_strides_0"), val = tensor([1])]; - tensor inputs_57_pad_0 = const()[name = tensor("inputs_57_pad_0"), val = tensor([0, 0])]; - tensor inputs_57_dilations_0 = const()[name = tensor("inputs_57_dilations_0"), val = tensor([1])]; - tensor inputs_57_groups_0 = const()[name = tensor("inputs_57_groups_0"), val = tensor(1)]; - tensor input_183 = transpose(perm = input_183_perm_0, x = x_31)[name = tensor("transpose_32")]; - tensor inputs_57 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_57_dilations_0, groups = inputs_57_groups_0, pad = inputs_57_pad_0, pad_type = inputs_57_pad_type_0, strides = inputs_57_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_183)[name = tensor("inputs_57")]; - tensor var_1250_split_sizes_0 = const()[name = tensor("op_1250_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1250_axis_0 = const()[name = tensor("op_1250_axis_0"), val = tensor(1)]; - tensor var_1250_0, tensor var_1250_1 = split(axis = var_1250_axis_0, split_sizes = var_1250_split_sizes_0, x = inputs_57)[name = tensor("op_1250")]; - tensor var_1252 = sigmoid(x = var_1250_1)[name = tensor("op_1252")]; - tensor inputs_59 = mul(x = var_1250_0, y = var_1252)[name = tensor("inputs_59")]; + tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_989)[name = tensor("x_expanded")]; + tensor var_1003 = sub(x = var_17, y = fill_mask)[name = tensor("op_1003")]; + tensor var_1004 = mul(x = window_21, y = var_1003)[name = tensor("op_1004")]; + tensor var_1005 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1005")]; + tensor filled_window = add(x = var_1004, y = var_1005)[name = tensor("filled_window")]; + tensor var_1007 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_1007")]; + tensor var_1008_dtype_0 = const()[name = tensor("op_1008_dtype_0"), val = tensor("fp32")]; + tensor var_1009 = const()[name = tensor("op_1009"), val = tensor([1, 1, 1])]; + tensor var_1008 = cast(dtype = var_1008_dtype_0, x = var_1007)[name = tensor("cast_128")]; + tensor use_roll = reshape(shape = var_1009, x = var_1008)[name = tensor("use_roll")]; + tensor var_1011 = sub(x = var_17, y = use_roll)[name = tensor("op_1011")]; + tensor var_1012 = mul(x = filled_window, y = var_1011)[name = tensor("op_1012")]; + tensor var_1013 = mul(x = rolled, y = use_roll)[name = tensor("op_1013")]; + tensor window = add(x = var_1012, y = var_1013)[name = tensor("window")]; + tensor var_1015 = const()[name = tensor("op_1015"), val = tensor(0x1p+0)]; + tensor var_1016 = add(x = clip_13, y = var_1015)[name = tensor("op_1016")]; + tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_1016)[name = tensor("clip_15")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_30, interleave = input_141_interleave_0, values = (window_21, window))[name = tensor("input_141")]; + tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; + tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; + tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_33_pad_type_0 = const()[name = tensor("inputs_33_pad_type_0"), val = tensor("valid")]; + tensor inputs_33_strides_0 = const()[name = tensor("inputs_33_strides_0"), val = tensor([1])]; + tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; + tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; + tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; + tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; + tensor var_1039_split_sizes_0 = const()[name = tensor("op_1039_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1039_axis_0 = const()[name = tensor("op_1039_axis_0"), val = tensor(1)]; + tensor var_1039_0, tensor var_1039_1 = split(axis = var_1039_axis_0, split_sizes = var_1039_split_sizes_0, x = inputs_33)[name = tensor("op_1039")]; + tensor var_1041 = sigmoid(x = var_1039_1)[name = tensor("op_1041")]; + tensor inputs_35 = mul(x = var_1039_0, y = var_1041)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; tensor outputs_aug_strides_0 = const()[name = tensor("outputs_aug_strides_0"), val = tensor([1])]; tensor outputs_aug_dilations_0 = const()[name = tensor("outputs_aug_dilations_0"), val = tensor([1])]; - tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_59)[name = tensor("outputs_aug")]; - tensor input_185_begin_0 = const()[name = tensor("input_185_begin_0"), val = tensor([0, 0, 0])]; - tensor input_185_end_0 = const()[name = tensor("input_185_end_0"), val = tensor([1, 256, 16])]; - tensor input_185_end_mask_0 = const()[name = tensor("input_185_end_mask_0"), val = tensor([true, true, false])]; - tensor input_185 = slice_by_index(begin = input_185_begin_0, end = input_185_end_0, end_mask = input_185_end_mask_0, x = outputs_aug)[name = tensor("input_185")]; - tensor inputs_61 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_185)[name = tensor("inputs_61")]; - tensor input_187 = silu(x = inputs_61)[name = tensor("input_187")]; - tensor input_189_pad_type_0 = const()[name = tensor("input_189_pad_type_0"), val = tensor("valid")]; - tensor input_189_strides_0 = const()[name = tensor("input_189_strides_0"), val = tensor([1])]; - tensor input_189_pad_0 = const()[name = tensor("input_189_pad_0"), val = tensor([0, 0])]; - tensor input_189_dilations_0 = const()[name = tensor("input_189_dilations_0"), val = tensor([1])]; - tensor input_189_groups_0 = const()[name = tensor("input_189_groups_0"), val = tensor(1)]; - tensor input_189 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_189_dilations_0, groups = input_189_groups_0, pad = input_189_pad_0, pad_type = input_189_pad_type_0, strides = input_189_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_187)[name = tensor("input_189")]; - tensor conv_out_15_perm_0 = const()[name = tensor("conv_out_15_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1283_begin_0 = const()[name = tensor("op_1283_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1283_end_0 = const()[name = tensor("op_1283_end_0"), val = tensor([1, 16, 256])]; - tensor var_1283_end_mask_0 = const()[name = tensor("op_1283_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_15 = transpose(perm = conv_out_15_perm_0, x = input_189)[name = tensor("transpose_31")]; - tensor var_1283 = slice_by_index(begin = var_1283_begin_0, end = var_1283_end_0, end_mask = var_1283_end_mask_0, x = conv_out_15)[name = tensor("op_1283")]; - tensor var_1286_interleave_0 = const()[name = tensor("op_1286_interleave_0"), val = tensor(false)]; - tensor var_1286 = concat(axis = var_33, interleave = var_1286_interleave_0, values = (var_1199, var_1283))[name = tensor("op_1286")]; - tensor input_191 = add(x = x_27, y = var_1286)[name = tensor("input_191")]; - tensor input_193_axes_0 = const()[name = tensor("input_193_axes_0"), val = tensor([-1])]; - tensor input_193 = layer_norm(axes = input_193_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_191)[name = tensor("input_193")]; - tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_193)[name = tensor("linear_35")]; - tensor input_195 = silu(x = inputs)[name = tensor("input_195")]; - tensor input_199 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_195)[name = tensor("linear_36")]; - tensor var_1309 = const()[name = tensor("op_1309"), val = tensor(0x1p-1)]; - tensor var_1310 = mul(x = input_199, y = var_1309)[name = tensor("op_1310")]; - tensor input_201 = add(x = var_1310, y = input_191)[name = tensor("input_201")]; - tensor x_33_axes_0 = const()[name = tensor("x_33_axes_0"), val = tensor([-1])]; - tensor x_33 = layer_norm(axes = x_33_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_201)[name = tensor("x_33")]; + tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_35)[name = tensor("outputs_aug")]; + tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; + tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([2, 256, 16])]; + tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; + tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug)[name = tensor("input_145")]; + tensor inputs_37 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_145)[name = tensor("inputs_37")]; + tensor input_147 = silu(x = inputs_37)[name = tensor("input_147")]; + tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; + tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; + tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; + tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; + tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; + tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; + tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1072_begin_0 = const()[name = tensor("op_1072_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1072_end_0 = const()[name = tensor("op_1072_end_0"), val = tensor([2, 16, 256])]; + tensor var_1072_end_mask_0 = const()[name = tensor("op_1072_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_1072 = slice_by_index(begin = var_1072_begin_0, end = var_1072_end_0, end_mask = var_1072_end_mask_0, x = conv_out_7)[name = tensor("op_1072")]; + tensor var_1074_perm_0 = const()[name = tensor("op_1074_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1074 = transpose(perm = var_1074_perm_0, x = var_1072)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_1074)[name = tensor("input_151")]; + tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; + tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; + tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; + tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; + tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; + tensor var_1097 = const()[name = tensor("op_1097"), val = tensor(0x1p-1)]; + tensor var_1098 = mul(x = input_159, y = var_1097)[name = tensor("op_1098")]; + tensor input_161 = add(x = var_1098, y = input_151)[name = tensor("input_161")]; + tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; + tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; tensor cat_interleave_0 = const()[name = tensor("cat_interleave_0"), val = tensor(false)]; - tensor x_ct = transpose(perm = x_ct_perm_0, x = x_33)[name = tensor("transpose_30")]; + tensor x_ct = transpose(perm = x_ct_perm_0, x = x_25)[name = tensor("transpose_30")]; tensor cat = concat(axis = var_27, interleave = cat_interleave_0, values = (cnn_window, x_ct))[name = tensor("cat")]; tensor conv_out_pad_type_0 = const()[name = tensor("conv_out_pad_type_0"), val = tensor("valid")]; tensor conv_out_strides_0 = const()[name = tensor("conv_out_strides_0"), val = tensor([1])]; @@ -1221,44 +1074,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_1328_begin_0 = const()[name = tensor("op_1328_begin_0"), val = tensor([0, 0, 2])]; - tensor var_1328_end_0 = const()[name = tensor("op_1328_end_0"), val = tensor([1, 256, 20])]; - tensor var_1328_end_mask_0 = const()[name = tensor("op_1328_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_1328_begin_0, end = var_1328_end_0, end_mask = var_1328_end_mask_0, x = cat)[name = tensor("op_1328")]; - tensor input_203_perm_0 = const()[name = tensor("input_203_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1330 = const()[name = tensor("op_1330"), val = tensor([-1])]; - tensor input_203 = transpose(perm = input_203_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_1331 = reduce_l2_norm(axes = var_1330, keep_dims = var_36, x = input_203)[name = tensor("op_1331")]; - tensor const_16 = const()[name = tensor("const_16"), val = tensor(0x1.fffffep+127)]; - tensor clip_16 = clip(alpha = var_12, beta = const_16, x = var_1331)[name = tensor("clip_16")]; - tensor emb = real_div(x = input_203, y = clip_16)[name = tensor("emb")]; - tensor var_1335_axis_0 = const()[name = tensor("op_1335_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_1335_axis_0, values = (var_214, var_510, var_806, nkv_1))[name = tensor("op_1335")]; - tensor var_1337_axis_0 = const()[name = tensor("op_1337_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_1337_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1337")]; - tensor var_1339_axis_0 = const()[name = tensor("op_1339_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_1339_axis_0, values = (input_31, input_81, input_131, input_181))[name = tensor("op_1339")]; - tensor var_1341_axis_0 = const()[name = tensor("op_1341_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_1341_axis_0, values = (clip_3, clip_7, clip_11, clip_15))[name = tensor("op_1341")]; - tensor var_1351 = const()[name = tensor("op_1351"), val = tensor(0x1.5798eep-27)]; - tensor var_1357 = const()[name = tensor("op_1357"), val = tensor(0x1.4f8b58p-17)]; - tensor var_1359 = const()[name = tensor("op_1359"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_1360 = const()[name = tensor("op_1360"), val = tensor(true)]; - tensor var_1362 = const()[name = tensor("op_1362"), val = tensor(0x1p+0)]; - tensor var_1366 = const()[name = tensor("op_1366"), val = tensor(-1)]; - tensor var_1371 = const()[name = tensor("op_1371"), val = tensor(0)]; + tensor var_1116_begin_0 = const()[name = tensor("op_1116_begin_0"), val = tensor([0, 0, 2])]; + tensor var_1116_end_0 = const()[name = tensor("op_1116_end_0"), val = tensor([1, 256, 20])]; + tensor var_1116_end_mask_0 = const()[name = tensor("op_1116_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_1116_begin_0, end = var_1116_end_0, end_mask = var_1116_end_mask_0, x = cat)[name = tensor("op_1116")]; + tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1118 = const()[name = tensor("op_1118"), val = tensor([-1])]; + tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; + tensor var_1119 = reduce_l2_norm(axes = var_1118, keep_dims = var_36, x = input_163)[name = tensor("op_1119")]; + tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; + tensor clip_16 = clip(alpha = var_12, beta = const_12, x = var_1119)[name = tensor("clip_16")]; + tensor emb = real_div(x = input_163, y = clip_16)[name = tensor("emb")]; + tensor var_1123_axis_0 = const()[name = tensor("op_1123_axis_0"), val = tensor(0)]; + tensor enc_kv_new = stack(axis = var_1123_axis_0, values = (var_214, var_457, var_700, nkv_1))[name = tensor("op_1123")]; + tensor var_1125_axis_0 = const()[name = tensor("op_1125_axis_0"), val = tensor(0)]; + tensor enc_scale_new = stack(axis = var_1125_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1125")]; + tensor var_1127_axis_0 = const()[name = tensor("op_1127_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1127_axis_0, values = (window_5, window_11, window_17, window))[name = tensor("op_1127")]; + tensor var_1129_axis_0 = const()[name = tensor("op_1129_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1129_axis_0, values = (clip_3, clip_7, clip_11, clip_15))[name = tensor("op_1129")]; + tensor var_1139 = const()[name = tensor("op_1139"), val = tensor(0x1.5798eep-27)]; + tensor var_1144 = const()[name = tensor("op_1144"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1146 = const()[name = tensor("op_1146"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1147 = const()[name = tensor("op_1147"), val = tensor(true)]; + tensor var_1149 = const()[name = tensor("op_1149"), val = tensor(0x1p+0)]; + tensor var_1153 = const()[name = tensor("op_1153"), val = tensor(-1)]; + tensor var_1159 = const()[name = tensor("op_1159"), val = tensor(0)]; tensor pos = const()[name = tensor("pos"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395456)))]; - tensor var_1433_axes_0 = const()[name = tensor("op_1433_axes_0"), val = tensor([2])]; - tensor var_1433 = expand_dims(axes = var_1433_axes_0, x = emb)[name = tensor("op_1433")]; - tensor var_1434 = const()[name = tensor("op_1434"), val = tensor([1, 1, 9, 1])]; - tensor emb_exp = tile(reps = var_1434, x = var_1433)[name = tensor("emb_exp")]; - tensor input_205_interleave_0 = const()[name = tensor("input_205_interleave_0"), val = tensor(false)]; - tensor input_205 = concat(axis = var_1366, interleave = input_205_interleave_0, values = (emb_exp, pos))[name = tensor("input_205")]; - tensor x_35 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_205)[name = tensor("linear_37")]; - tensor var_1441_perm_0 = const()[name = tensor("op_1441_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1445 = const()[name = tensor("op_1445"), val = tensor([9, 2, 256])]; - tensor var_1441 = transpose(perm = var_1441_perm_0, x = x_35)[name = tensor("transpose_28")]; - tensor x_37 = reshape(shape = var_1445, x = var_1441)[name = tensor("x_37")]; + tensor var_1221_axes_0 = const()[name = tensor("op_1221_axes_0"), val = tensor([2])]; + tensor var_1221 = expand_dims(axes = var_1221_axes_0, x = emb)[name = tensor("op_1221")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 9, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1221)[name = tensor("emb_exp")]; + tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; + tensor input_165 = concat(axis = var_1153, interleave = input_165_interleave_0, values = (emb_exp, pos))[name = tensor("input_165")]; + tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; + tensor var_1229_perm_0 = const()[name = tensor("op_1229_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1233 = const()[name = tensor("op_1233"), val = tensor([9, 2, 256])]; + tensor var_1229 = transpose(perm = var_1229_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1233, x = var_1229)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 9, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1269,132 +1122,132 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_1453 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_37)[name = tensor("linear_38")]; - tensor var_1454 = const()[name = tensor("op_1454"), val = tensor([9, 2, 4, 64])]; - tensor var_1455 = reshape(shape = var_1454, x = var_1453)[name = tensor("op_1455")]; + tensor var_1241 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1242 = const()[name = tensor("op_1242"), val = tensor([9, 2, 4, 64])]; + tensor var_1243 = reshape(shape = var_1242, x = var_1241)[name = tensor("op_1243")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1459 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_37)[name = tensor("linear_39")]; - tensor var_1460 = const()[name = tensor("op_1460"), val = tensor(0x1p-3)]; - tensor var_1461 = mul(x = var_1459, y = var_1460)[name = tensor("op_1461")]; - tensor var_1462 = const()[name = tensor("op_1462"), val = tensor([9, 2, 4, 64])]; - tensor var_1463 = reshape(shape = var_1462, x = var_1461)[name = tensor("op_1463")]; + tensor var_1247 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1248 = const()[name = tensor("op_1248"), val = tensor(0x1p-3)]; + tensor var_1249 = mul(x = var_1247, y = var_1248)[name = tensor("op_1249")]; + tensor var_1250 = const()[name = tensor("op_1250"), val = tensor([9, 2, 4, 64])]; + tensor var_1251 = reshape(shape = var_1250, x = var_1249)[name = tensor("op_1251")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1467 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_37)[name = tensor("linear_40")]; - tensor var_1468 = const()[name = tensor("op_1468"), val = tensor([9, 2, 4, 64])]; - tensor var_1469 = reshape(shape = var_1468, x = var_1467)[name = tensor("op_1469")]; + tensor var_1255 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1256 = const()[name = tensor("op_1256"), val = tensor([9, 2, 4, 64])]; + tensor var_1257 = reshape(shape = var_1256, x = var_1255)[name = tensor("op_1257")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_209 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_37)[name = tensor("linear_41")]; + tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_1371, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1159, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; - tensor const_24 = const()[name = tensor("const_24"), val = tensor(0x1.fffffep+127)]; - tensor clip_17 = clip(alpha = var_1362, beta = const_24, x = s_t_9)[name = tensor("clip_17")]; + tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; + tensor clip_17 = clip(alpha = var_1149, beta = const_20, x = s_t_9)[name = tensor("clip_17")]; tensor sqrt_s_t_9 = sqrt(x = clip_17)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_1463)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_1455)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1251)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1243)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_1481 = const()[name = tensor("op_1481"), val = tensor([1, 2])]; - tensor var_1482 = reshape(shape = var_1481, x = valid_mask)[name = tensor("op_1482")]; - tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1482)[name = tensor("causal_with_valid_1")]; - tensor var_1484 = const()[name = tensor("op_1484"), val = tensor([2, 1])]; - tensor var_1485 = reshape(shape = var_1484, x = sqrt_s_t_9)[name = tensor("op_1485")]; - tensor M_9 = real_div(x = causal_with_valid_1, y = var_1485)[name = tensor("M_9")]; - tensor var_1487 = mul(x = qk_9, y = M_9)[name = tensor("op_1487")]; + tensor var_1269 = const()[name = tensor("op_1269"), val = tensor([1, 2])]; + tensor var_1270 = reshape(shape = var_1269, x = valid_mask)[name = tensor("op_1270")]; + tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1270)[name = tensor("causal_with_valid_1")]; + tensor var_1272 = const()[name = tensor("op_1272"), val = tensor([2, 1])]; + tensor var_1273 = reshape(shape = var_1272, x = sqrt_s_t_9)[name = tensor("op_1273")]; + tensor M_9 = real_div(x = causal_with_valid_1, y = var_1273)[name = tensor("M_9")]; + tensor var_1275 = mul(x = qk_9, y = M_9)[name = tensor("op_1275")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_1469)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1487, y = v_9)[name = tensor("inner_9")]; - tensor var_1489_transpose_x_0 = const()[name = tensor("op_1489_transpose_x_0"), val = tensor(false)]; - tensor var_1489_transpose_y_0 = const()[name = tensor("op_1489_transpose_y_0"), val = tensor(false)]; - tensor var_1489 = matmul(transpose_x = var_1489_transpose_x_0, transpose_y = var_1489_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1489")]; - tensor var_1490 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1490")]; - tensor var_1491 = const()[name = tensor("op_1491"), val = tensor([1, 1, 2, 1])]; - tensor var_1492 = reshape(shape = var_1491, x = var_1490)[name = tensor("op_1492")]; - tensor cross_9 = mul(x = var_1489, y = var_1492)[name = tensor("cross_9")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1257)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1275, y = v_9)[name = tensor("inner_9")]; + tensor var_1277_transpose_x_0 = const()[name = tensor("op_1277_transpose_x_0"), val = tensor(false)]; + tensor var_1277_transpose_y_0 = const()[name = tensor("op_1277_transpose_y_0"), val = tensor(false)]; + tensor var_1277 = matmul(transpose_x = var_1277_transpose_x_0, transpose_y = var_1277_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1277")]; + tensor var_1278 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1278")]; + tensor var_1279 = const()[name = tensor("op_1279"), val = tensor([1, 1, 2, 1])]; + tensor var_1280 = reshape(shape = var_1279, x = var_1278)[name = tensor("op_1280")]; + tensor cross_9 = mul(x = var_1277, y = var_1280)[name = tensor("cross_9")]; tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; - tensor var_1495 = const()[name = tensor("op_1495"), val = tensor([1, 1, 2, 1])]; - tensor var_1496 = reshape(shape = var_1495, x = valid_mask)[name = tensor("op_1496")]; - tensor v_masked_1 = mul(x = v_9, y = var_1496)[name = tensor("v_masked_1")]; - tensor var_1498 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1498")]; - tensor var_1500_transpose_x_1 = const()[name = tensor("op_1500_transpose_x_1"), val = tensor(true)]; - tensor var_1500_transpose_y_1 = const()[name = tensor("op_1500_transpose_y_1"), val = tensor(false)]; - tensor var_1500 = matmul(transpose_x = var_1500_transpose_x_1, transpose_y = var_1500_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1500")]; - tensor new_kv_unnorm_9 = add(x = var_1498, y = var_1500)[name = tensor("new_kv_unnorm_9")]; - tensor var_1502_keep_dims_0 = const()[name = tensor("op_1502_keep_dims_0"), val = tensor(false)]; - tensor var_1502 = reduce_sum(keep_dims = var_1502_keep_dims_0, x = valid_mask)[name = tensor("op_1502")]; - tensor var_1503 = const()[name = tensor("op_1503"), val = tensor([1])]; - tensor var_1504 = reshape(shape = var_1503, x = var_1502)[name = tensor("op_1504")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_1504)[name = tensor("new_scale_9")]; - tensor const_25 = const()[name = tensor("const_25"), val = tensor(0x1.fffffep+127)]; - tensor clip_18 = clip(alpha = var_1362, beta = const_25, x = new_scale_9)[name = tensor("clip_18")]; + tensor var_1283 = const()[name = tensor("op_1283"), val = tensor([1, 1, 2, 1])]; + tensor var_1284 = reshape(shape = var_1283, x = valid_mask)[name = tensor("op_1284")]; + tensor v_masked_1 = mul(x = v_9, y = var_1284)[name = tensor("v_masked_1")]; + tensor var_1286 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1286")]; + tensor var_1288_transpose_x_1 = const()[name = tensor("op_1288_transpose_x_1"), val = tensor(true)]; + tensor var_1288_transpose_y_1 = const()[name = tensor("op_1288_transpose_y_1"), val = tensor(false)]; + tensor var_1288 = matmul(transpose_x = var_1288_transpose_x_1, transpose_y = var_1288_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1288")]; + tensor new_kv_unnorm_9 = add(x = var_1286, y = var_1288)[name = tensor("new_kv_unnorm_9")]; + tensor var_1290_keep_dims_0 = const()[name = tensor("op_1290_keep_dims_0"), val = tensor(false)]; + tensor var_1290 = reduce_sum(keep_dims = var_1290_keep_dims_0, x = valid_mask)[name = tensor("op_1290")]; + tensor var_1291 = const()[name = tensor("op_1291"), val = tensor([1])]; + tensor var_1292 = reshape(shape = var_1291, x = var_1290)[name = tensor("op_1292")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1292)[name = tensor("new_scale_9")]; + tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; + tensor clip_18 = clip(alpha = var_1149, beta = const_21, x = new_scale_9)[name = tensor("clip_18")]; tensor sqrt_new_scale_1 = sqrt(x = clip_18)[name = tensor("sqrt_new_scale_1")]; - tensor var_1508 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1508")]; - tensor var_1509_perm_0 = const()[name = tensor("op_1509_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1296 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1296")]; + tensor var_1297_perm_0 = const()[name = tensor("op_1297_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_1509 = transpose(perm = var_1509_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1359, x = var_1509)[name = tensor("out_27")]; - tensor var_1513 = const()[name = tensor("op_1513"), val = tensor([9, 2, 256])]; - tensor out_29 = reshape(shape = var_1513, x = out_27)[name = tensor("out_29")]; - tensor var_1515 = silu(x = input_209)[name = tensor("op_1515")]; - tensor input_211 = mul(x = var_1515, y = out_29)[name = tensor("input_211")]; - tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_211)[name = tensor("linear_42")]; - tensor input_213 = add(x = x_37, y = ret_out_9)[name = tensor("input_213")]; + tensor var_1297 = transpose(perm = var_1297_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1146, x = var_1297)[name = tensor("out_27")]; + tensor var_1301 = const()[name = tensor("op_1301"), val = tensor([9, 2, 256])]; + tensor out_29 = reshape(shape = var_1301, x = out_27)[name = tensor("out_29")]; + tensor var_1303 = silu(x = input_169)[name = tensor("op_1303")]; + tensor input_171 = mul(x = var_1303, y = out_29)[name = tensor("input_171")]; + tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; + tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1357, gamma = decoder_norm11_0_weight, x = input_213)[name = tensor("xt_1")]; - tensor var_1525 = const()[name = tensor("op_1525"), val = tensor([1, 9, 2, 256])]; - tensor var_1526 = reshape(shape = var_1525, x = xt_1)[name = tensor("op_1526")]; - tensor var_1527_perm_0 = const()[name = tensor("op_1527_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1530 = const()[name = tensor("op_1530"), val = tensor([2, 9, 256])]; - tensor var_1527 = transpose(perm = var_1527_perm_0, x = var_1526)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_1530, x = var_1527)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1144, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1313 = const()[name = tensor("op_1313"), val = tensor([1, 9, 2, 256])]; + tensor var_1314 = reshape(shape = var_1313, x = xt_1)[name = tensor("op_1314")]; + tensor var_1315_perm_0 = const()[name = tensor("op_1315_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1318 = const()[name = tensor("op_1318"), val = tensor([2, 9, 256])]; + tensor var_1315 = transpose(perm = var_1315_perm_0, x = var_1314)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1318, x = var_1315)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_1553 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_8 = const()[name = tensor("concat_8"), val = tensor([9, 2, 3, 256])]; - tensor var_1555 = reshape(shape = concat_8, x = var_1553)[name = tensor("op_1555")]; - tensor var_1556_axes_0 = const()[name = tensor("op_1556_axes_0"), val = tensor([0])]; - tensor var_1556 = expand_dims(axes = var_1556_axes_0, x = var_1555)[name = tensor("op_1556")]; - tensor var_1557_perm_0 = const()[name = tensor("op_1557_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1558_axes_0 = const()[name = tensor("op_1558_axes_0"), val = tensor([-2])]; - tensor var_1557 = transpose(perm = var_1557_perm_0, x = var_1556)[name = tensor("transpose_21")]; - tensor var_1558 = squeeze(axes = var_1558_axes_0, x = var_1557)[name = tensor("op_1558")]; + tensor var_1341 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_9 = const()[name = tensor("concat_9"), val = tensor([9, 2, 3, 256])]; + tensor var_1343 = reshape(shape = concat_9, x = var_1341)[name = tensor("op_1343")]; + tensor var_1344_axes_0 = const()[name = tensor("op_1344_axes_0"), val = tensor([0])]; + tensor var_1344 = expand_dims(axes = var_1344_axes_0, x = var_1343)[name = tensor("op_1344")]; + tensor var_1345_perm_0 = const()[name = tensor("op_1345_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1346_axes_0 = const()[name = tensor("op_1346_axes_0"), val = tensor([-2])]; + tensor var_1345 = transpose(perm = var_1345_perm_0, x = var_1344)[name = tensor("transpose_21")]; + tensor var_1346 = squeeze(axes = var_1346_axes_0, x = var_1345)[name = tensor("op_1346")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 9, 2, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1558)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1346)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 9, 2, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1558)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1346)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 9, 2, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1558)[name = tensor("v_11")]; - tensor var_1566 = const()[name = tensor("op_1566"), val = tensor([9, 8, 64])]; - tensor var_1567 = reshape(shape = var_1566, x = q_11)[name = tensor("op_1567")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1346)[name = tensor("v_11")]; + tensor var_1354 = const()[name = tensor("op_1354"), val = tensor([9, 8, 64])]; + tensor var_1355 = reshape(shape = var_1354, x = q_11)[name = tensor("op_1355")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1573 = const()[name = tensor("op_1573"), val = tensor([9, 8, 64])]; - tensor var_1574 = reshape(shape = var_1573, x = k_11)[name = tensor("op_1574")]; + tensor var_1361 = const()[name = tensor("op_1361"), val = tensor([9, 8, 64])]; + tensor var_1362 = reshape(shape = var_1361, x = k_11)[name = tensor("op_1362")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1580 = const()[name = tensor("op_1580"), val = tensor([9, 8, 64])]; - tensor var_1581 = reshape(shape = var_1580, x = v_11)[name = tensor("op_1581")]; + tensor var_1368 = const()[name = tensor("op_1368"), val = tensor([9, 8, 64])]; + tensor var_1369 = reshape(shape = var_1368, x = v_11)[name = tensor("op_1369")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1584 = const()[name = tensor("op_1584"), val = tensor([2, 4, 9, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_1567)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_1584, x = q_13)[name = tensor("q_15")]; - tensor var_1586 = const()[name = tensor("op_1586"), val = tensor([2, 4, 9, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_1574)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_1586, x = k_13)[name = tensor("k_15")]; - tensor var_1588 = const()[name = tensor("op_1588"), val = tensor([2, 4, 9, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_1581)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_1588, x = v_13)[name = tensor("v_15")]; + tensor var_1372 = const()[name = tensor("op_1372"), val = tensor([2, 4, 9, 64])]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1355)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1372, x = q_13)[name = tensor("q_15")]; + tensor var_1374 = const()[name = tensor("op_1374"), val = tensor([2, 4, 9, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1362)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1374, x = k_13)[name = tensor("k_15")]; + tensor var_1376 = const()[name = tensor("op_1376"), val = tensor([2, 4, 9, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1369)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1376, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -1405,30 +1258,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_1591 = const()[name = tensor("op_1591"), val = tensor([2, 0, 1, 3])]; - tensor var_1596 = const()[name = tensor("op_1596"), val = tensor([18, 256])]; - tensor var_1592 = transpose(perm = var_1591, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_1596, x = var_1592)[name = tensor("attn_output_3")]; + tensor var_1379 = const()[name = tensor("op_1379"), val = tensor([2, 0, 1, 3])]; + tensor var_1384 = const()[name = tensor("op_1384"), val = tensor([18, 256])]; + tensor var_1380 = transpose(perm = var_1379, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1384, x = var_1380)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_1600 = const()[name = tensor("op_1600"), val = tensor([9, 2, 256])]; - tensor attn_output_7 = reshape(shape = var_1600, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1388 = const()[name = tensor("op_1388"), val = tensor([9, 2, 256])]; + tensor attn_output_7 = reshape(shape = var_1388, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; - tensor input_215 = add(x = query_1, y = sa2_out_1)[name = tensor("input_215")]; - tensor input_217_axes_0 = const()[name = tensor("input_217_axes_0"), val = tensor([-1])]; - tensor input_217 = layer_norm(axes = input_217_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1357, gamma = decoder_norm21_0_weight, x = input_215)[name = tensor("input_217")]; - tensor input_219 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_217)[name = tensor("linear_45")]; - tensor input_221 = relu(x = input_219)[name = tensor("input_221")]; - tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_221)[name = tensor("linear_46")]; - tensor input_223 = add(x = input_217, y = ffn_out_1)[name = tensor("input_223")]; + tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; + tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1144, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; + tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; + tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; + tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1357, gamma = decoder_norm22_0_weight, x = input_223)[name = tensor("xt_3")]; - tensor var_1620 = const()[name = tensor("op_1620"), val = tensor([1, 2, 9, 256])]; - tensor x_39 = reshape(shape = var_1620, x = xt_3)[name = tensor("x_39")]; - tensor var_1622_perm_0 = const()[name = tensor("op_1622_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1626 = const()[name = tensor("op_1626"), val = tensor([9, 2, 256])]; - tensor var_1622 = transpose(perm = var_1622_perm_0, x = x_39)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_1626, x = var_1622)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1144, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1408 = const()[name = tensor("op_1408"), val = tensor([1, 2, 9, 256])]; + tensor x_31 = reshape(shape = var_1408, x = xt_3)[name = tensor("x_31")]; + tensor var_1410_perm_0 = const()[name = tensor("op_1410_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1414 = const()[name = tensor("op_1414"), val = tensor([9, 2, 256])]; + tensor var_1410 = transpose(perm = var_1410_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1414, x = var_1410)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 9, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1439,120 +1292,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_1634 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_1635 = const()[name = tensor("op_1635"), val = tensor([9, 2, 4, 64])]; - tensor var_1636 = reshape(shape = var_1635, x = var_1634)[name = tensor("op_1636")]; + tensor var_1422 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1423 = const()[name = tensor("op_1423"), val = tensor([9, 2, 4, 64])]; + tensor var_1424 = reshape(shape = var_1423, x = var_1422)[name = tensor("op_1424")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1640 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_1641 = const()[name = tensor("op_1641"), val = tensor(0x1p-3)]; - tensor var_1642 = mul(x = var_1640, y = var_1641)[name = tensor("op_1642")]; - tensor var_1643 = const()[name = tensor("op_1643"), val = tensor([9, 2, 4, 64])]; - tensor var_1644 = reshape(shape = var_1643, x = var_1642)[name = tensor("op_1644")]; + tensor var_1428 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1429 = const()[name = tensor("op_1429"), val = tensor(0x1p-3)]; + tensor var_1430 = mul(x = var_1428, y = var_1429)[name = tensor("op_1430")]; + tensor var_1431 = const()[name = tensor("op_1431"), val = tensor([9, 2, 4, 64])]; + tensor var_1432 = reshape(shape = var_1431, x = var_1430)[name = tensor("op_1432")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1648 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_1649 = const()[name = tensor("op_1649"), val = tensor([9, 2, 4, 64])]; - tensor var_1650 = reshape(shape = var_1649, x = var_1648)[name = tensor("op_1650")]; + tensor var_1436 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1437 = const()[name = tensor("op_1437"), val = tensor([9, 2, 4, 64])]; + tensor var_1438 = reshape(shape = var_1437, x = var_1436)[name = tensor("op_1438")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_227 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; + tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; - tensor const_36 = const()[name = tensor("const_36"), val = tensor(0x1.fffffep+127)]; - tensor clip_19 = clip(alpha = var_1362, beta = const_36, x = s_t)[name = tensor("clip_19")]; + tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; + tensor clip_19 = clip(alpha = var_1149, beta = const_32, x = s_t)[name = tensor("clip_19")]; tensor sqrt_s_t = sqrt(x = clip_19)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_1644)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_1636)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1432)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1424)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_1665 = const()[name = tensor("op_1665"), val = tensor([2, 1])]; - tensor var_1666 = reshape(shape = var_1665, x = sqrt_s_t)[name = tensor("op_1666")]; - tensor M = real_div(x = causal_with_valid_1, y = var_1666)[name = tensor("M")]; - tensor var_1668 = mul(x = qk, y = M)[name = tensor("op_1668")]; + tensor var_1453 = const()[name = tensor("op_1453"), val = tensor([2, 1])]; + tensor var_1454 = reshape(shape = var_1453, x = sqrt_s_t)[name = tensor("op_1454")]; + tensor M = real_div(x = causal_with_valid_1, y = var_1454)[name = tensor("M")]; + tensor var_1456 = mul(x = qk, y = M)[name = tensor("op_1456")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_1650)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1668, y = v_17)[name = tensor("inner")]; - tensor var_1670_transpose_x_0 = const()[name = tensor("op_1670_transpose_x_0"), val = tensor(false)]; - tensor var_1670_transpose_y_0 = const()[name = tensor("op_1670_transpose_y_0"), val = tensor(false)]; - tensor var_1670 = matmul(transpose_x = var_1670_transpose_x_0, transpose_y = var_1670_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1670")]; - tensor var_1671 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1671")]; - tensor var_1672 = const()[name = tensor("op_1672"), val = tensor([1, 1, 2, 1])]; - tensor var_1673 = reshape(shape = var_1672, x = var_1671)[name = tensor("op_1673")]; - tensor cross = mul(x = var_1670, y = var_1673)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1438)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1456, y = v_17)[name = tensor("inner")]; + tensor var_1458_transpose_x_0 = const()[name = tensor("op_1458_transpose_x_0"), val = tensor(false)]; + tensor var_1458_transpose_y_0 = const()[name = tensor("op_1458_transpose_y_0"), val = tensor(false)]; + tensor var_1458 = matmul(transpose_x = var_1458_transpose_x_0, transpose_y = var_1458_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1458")]; + tensor var_1459 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1459")]; + tensor var_1460 = const()[name = tensor("op_1460"), val = tensor([1, 1, 2, 1])]; + tensor var_1461 = reshape(shape = var_1460, x = var_1459)[name = tensor("op_1461")]; + tensor cross = mul(x = var_1458, y = var_1461)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_1496)[name = tensor("v_masked")]; - tensor var_1679 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1679")]; - tensor var_1681_transpose_x_1 = const()[name = tensor("op_1681_transpose_x_1"), val = tensor(true)]; - tensor var_1681_transpose_y_1 = const()[name = tensor("op_1681_transpose_y_1"), val = tensor(false)]; - tensor var_1681 = matmul(transpose_x = var_1681_transpose_x_1, transpose_y = var_1681_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1681")]; - tensor new_kv_unnorm = add(x = var_1679, y = var_1681)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_1504)[name = tensor("new_scale")]; - tensor const_37 = const()[name = tensor("const_37"), val = tensor(0x1.fffffep+127)]; - tensor clip_20 = clip(alpha = var_1362, beta = const_37, x = new_scale)[name = tensor("clip_20")]; + tensor v_masked = mul(x = v_17, y = var_1284)[name = tensor("v_masked")]; + tensor var_1467 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1467")]; + tensor var_1469_transpose_x_1 = const()[name = tensor("op_1469_transpose_x_1"), val = tensor(true)]; + tensor var_1469_transpose_y_1 = const()[name = tensor("op_1469_transpose_y_1"), val = tensor(false)]; + tensor var_1469 = matmul(transpose_x = var_1469_transpose_x_1, transpose_y = var_1469_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1469")]; + tensor new_kv_unnorm = add(x = var_1467, y = var_1469)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1292)[name = tensor("new_scale")]; + tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; + tensor clip_20 = clip(alpha = var_1149, beta = const_33, x = new_scale)[name = tensor("clip_20")]; tensor sqrt_new_scale = sqrt(x = clip_20)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_1690_perm_0 = const()[name = tensor("op_1690_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1478_perm_0 = const()[name = tensor("op_1478_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_1690 = transpose(perm = var_1690_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1359, x = var_1690)[name = tensor("out_33")]; - tensor var_1694 = const()[name = tensor("op_1694"), val = tensor([9, 2, 256])]; - tensor out = reshape(shape = var_1694, x = out_33)[name = tensor("out")]; - tensor var_1696 = silu(x = input_227)[name = tensor("op_1696")]; - tensor input_229 = mul(x = var_1696, y = out)[name = tensor("input_229")]; - tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_229)[name = tensor("linear_51")]; - tensor input_231 = add(x = x, y = ret_out)[name = tensor("input_231")]; + tensor var_1478 = transpose(perm = var_1478_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1146, x = var_1478)[name = tensor("out_33")]; + tensor var_1482 = const()[name = tensor("op_1482"), val = tensor([9, 2, 256])]; + tensor out = reshape(shape = var_1482, x = out_33)[name = tensor("out")]; + tensor var_1484 = silu(x = input_187)[name = tensor("op_1484")]; + tensor input_189 = mul(x = var_1484, y = out)[name = tensor("input_189")]; + tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; + tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1357, gamma = decoder_norm11_1_weight, x = input_231)[name = tensor("xt_5")]; - tensor var_1706 = const()[name = tensor("op_1706"), val = tensor([1, 9, 2, 256])]; - tensor var_1707 = reshape(shape = var_1706, x = xt_5)[name = tensor("op_1707")]; - tensor var_1708_perm_0 = const()[name = tensor("op_1708_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1711 = const()[name = tensor("op_1711"), val = tensor([2, 9, 256])]; - tensor var_1708 = transpose(perm = var_1708_perm_0, x = var_1707)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_1711, x = var_1708)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1144, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1494 = const()[name = tensor("op_1494"), val = tensor([1, 9, 2, 256])]; + tensor var_1495 = reshape(shape = var_1494, x = xt_5)[name = tensor("op_1495")]; + tensor var_1496_perm_0 = const()[name = tensor("op_1496_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1499 = const()[name = tensor("op_1499"), val = tensor([2, 9, 256])]; + tensor var_1496 = transpose(perm = var_1496_perm_0, x = var_1495)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1499, x = var_1496)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_1734 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_9 = const()[name = tensor("concat_9"), val = tensor([9, 2, 3, 256])]; - tensor var_1736 = reshape(shape = concat_9, x = var_1734)[name = tensor("op_1736")]; - tensor var_1737_axes_0 = const()[name = tensor("op_1737_axes_0"), val = tensor([0])]; - tensor var_1737 = expand_dims(axes = var_1737_axes_0, x = var_1736)[name = tensor("op_1737")]; - tensor var_1738_perm_0 = const()[name = tensor("op_1738_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1739_axes_0 = const()[name = tensor("op_1739_axes_0"), val = tensor([-2])]; - tensor var_1738 = transpose(perm = var_1738_perm_0, x = var_1737)[name = tensor("transpose_8")]; - tensor var_1739 = squeeze(axes = var_1739_axes_0, x = var_1738)[name = tensor("op_1739")]; + tensor var_1522 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_10 = const()[name = tensor("concat_10"), val = tensor([9, 2, 3, 256])]; + tensor var_1524 = reshape(shape = concat_10, x = var_1522)[name = tensor("op_1524")]; + tensor var_1525_axes_0 = const()[name = tensor("op_1525_axes_0"), val = tensor([0])]; + tensor var_1525 = expand_dims(axes = var_1525_axes_0, x = var_1524)[name = tensor("op_1525")]; + tensor var_1526_perm_0 = const()[name = tensor("op_1526_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1527_axes_0 = const()[name = tensor("op_1527_axes_0"), val = tensor([-2])]; + tensor var_1526 = transpose(perm = var_1526_perm_0, x = var_1525)[name = tensor("transpose_8")]; + tensor var_1527 = squeeze(axes = var_1527_axes_0, x = var_1526)[name = tensor("op_1527")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 9, 2, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1739)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1527)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 9, 2, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1739)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1527)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 9, 2, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1739)[name = tensor("v_19")]; - tensor var_1747 = const()[name = tensor("op_1747"), val = tensor([9, 8, 64])]; - tensor var_1748 = reshape(shape = var_1747, x = q_19)[name = tensor("op_1748")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1527)[name = tensor("v_19")]; + tensor var_1535 = const()[name = tensor("op_1535"), val = tensor([9, 8, 64])]; + tensor var_1536 = reshape(shape = var_1535, x = q_19)[name = tensor("op_1536")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1754 = const()[name = tensor("op_1754"), val = tensor([9, 8, 64])]; - tensor var_1755 = reshape(shape = var_1754, x = k_19)[name = tensor("op_1755")]; + tensor var_1542 = const()[name = tensor("op_1542"), val = tensor([9, 8, 64])]; + tensor var_1543 = reshape(shape = var_1542, x = k_19)[name = tensor("op_1543")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1761 = const()[name = tensor("op_1761"), val = tensor([9, 8, 64])]; - tensor var_1762 = reshape(shape = var_1761, x = v_19)[name = tensor("op_1762")]; + tensor var_1549 = const()[name = tensor("op_1549"), val = tensor([9, 8, 64])]; + tensor var_1550 = reshape(shape = var_1549, x = v_19)[name = tensor("op_1550")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1765 = const()[name = tensor("op_1765"), val = tensor([2, 4, 9, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_1748)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_1765, x = q_21)[name = tensor("q")]; - tensor var_1767 = const()[name = tensor("op_1767"), val = tensor([2, 4, 9, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_1755)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_1767, x = k_21)[name = tensor("k")]; - tensor var_1769 = const()[name = tensor("op_1769"), val = tensor([2, 4, 9, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_1762)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_1769, x = v_21)[name = tensor("v")]; + tensor var_1553 = const()[name = tensor("op_1553"), val = tensor([2, 4, 9, 64])]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1536)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1553, x = q_21)[name = tensor("q")]; + tensor var_1555 = const()[name = tensor("op_1555"), val = tensor([2, 4, 9, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1543)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1555, x = k_21)[name = tensor("k")]; + tensor var_1557 = const()[name = tensor("op_1557"), val = tensor([2, 4, 9, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1550)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1557, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -1563,50 +1416,50 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_1772 = const()[name = tensor("op_1772"), val = tensor([2, 0, 1, 3])]; - tensor var_1777 = const()[name = tensor("op_1777"), val = tensor([18, 256])]; - tensor var_1773 = transpose(perm = var_1772, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_1777, x = var_1773)[name = tensor("attn_output_11")]; + tensor var_1560 = const()[name = tensor("op_1560"), val = tensor([2, 0, 1, 3])]; + tensor var_1565 = const()[name = tensor("op_1565"), val = tensor([18, 256])]; + tensor var_1561 = transpose(perm = var_1560, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1565, x = var_1561)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_1781 = const()[name = tensor("op_1781"), val = tensor([9, 2, 256])]; - tensor attn_output = reshape(shape = var_1781, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1569 = const()[name = tensor("op_1569"), val = tensor([9, 2, 256])]; + tensor attn_output = reshape(shape = var_1569, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; - tensor input_233 = add(x = query_5, y = sa2_out)[name = tensor("input_233")]; - tensor input_235_axes_0 = const()[name = tensor("input_235_axes_0"), val = tensor([-1])]; - tensor input_235 = layer_norm(axes = input_235_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1357, gamma = decoder_norm21_1_weight, x = input_233)[name = tensor("input_235")]; - tensor input_237 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_235)[name = tensor("linear_54")]; - tensor input_239 = relu(x = input_237)[name = tensor("input_239")]; - tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_239)[name = tensor("linear_55")]; - tensor input_241 = add(x = input_235, y = ffn_out)[name = tensor("input_241")]; + tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; + tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1144, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; + tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; + tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; + tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1357, gamma = decoder_norm22_1_weight, x = input_241)[name = tensor("xt")]; - tensor var_1801 = const()[name = tensor("op_1801"), val = tensor([1, 2, 9, 256])]; - tensor input = reshape(shape = var_1801, x = xt)[name = tensor("input")]; - tensor var_1803 = const()[name = tensor("op_1803"), val = tensor([-1])]; - tensor var_1804 = reduce_l2_norm(axes = var_1803, keep_dims = var_1360, x = input)[name = tensor("op_1804")]; - tensor const_46 = const()[name = tensor("const_46"), val = tensor(0x1.fffffep+127)]; - tensor clip_21 = clip(alpha = var_1351, beta = const_46, x = var_1804)[name = tensor("clip_21")]; - tensor var_1806 = real_div(x = input, y = clip_21)[name = tensor("op_1806")]; - tensor concat_13 = const()[name = tensor("concat_13"), val = tensor([2, 1, 256])]; - tensor reshape_0 = reshape(shape = concat_13, x = emb)[name = tensor("reshape_0")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1144, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1589 = const()[name = tensor("op_1589"), val = tensor([1, 2, 9, 256])]; + tensor input = reshape(shape = var_1589, x = xt)[name = tensor("input")]; + tensor var_1591 = const()[name = tensor("op_1591"), val = tensor([-1])]; + tensor var_1592 = reduce_l2_norm(axes = var_1591, keep_dims = var_1147, x = input)[name = tensor("op_1592")]; + tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; + tensor clip_21 = clip(alpha = var_1139, beta = const_42, x = var_1592)[name = tensor("clip_21")]; + tensor var_1594 = real_div(x = input, y = clip_21)[name = tensor("op_1594")]; + tensor concat_14 = const()[name = tensor("concat_14"), val = tensor([2, 1, 256])]; + tensor reshape_0 = reshape(shape = concat_14, x = emb)[name = tensor("reshape_0")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_14 = const()[name = tensor("concat_14"), val = tensor([2, 256, 9])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1806)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_14, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_15 = const()[name = tensor("concat_15"), val = tensor([2, 256, 9])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1594)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_15, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = reshape_0, y = reshape_1)[name = tensor("matmul_2")]; - tensor concat_17 = const()[name = tensor("concat_17"), val = tensor([1, 2, 9])]; - tensor reshape_2 = reshape(shape = concat_17, x = matmul_2)[name = tensor("reshape_2")]; + tensor concat_18 = const()[name = tensor("concat_18"), val = tensor([1, 2, 9])]; + tensor reshape_2 = reshape(shape = concat_18, x = matmul_2)[name = tensor("reshape_2")]; tensor output_begin_0 = const()[name = tensor("output_begin_0"), val = tensor([0, 0, 1])]; tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 2, 8])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = reshape_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_1810")]; - tensor var_1812_axis_0 = const()[name = tensor("op_1812_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_1812_axis_0, values = (var_1508, nkv))[name = tensor("op_1812")]; - tensor var_1814_axis_0 = const()[name = tensor("op_1814_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_1814_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1814")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1598")]; + tensor var_1600_axis_0 = const()[name = tensor("op_1600_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1600_axis_0, values = (var_1296, nkv))[name = tensor("op_1600")]; + tensor var_1602_axis_0 = const()[name = tensor("op_1602_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1602_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1602")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/CALLHOME/ls_eend_ch_200ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/CALLHOME/ls_eend_ch_200ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index 3f60aa9efc5e51a3bdd4daf7f3f99af57c32a7f0..848b10c8d9ac4d5736b9458aeed101d107367522 100644 --- a/optimized/CALLHOME/ls_eend_ch_200ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/CALLHOME/ls_eend_ch_200ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:95c3c28175e58b432bfc70c82d87157990f2167f1420230525d3359c3096b3c2 -size 228666 +oid sha256:adca28589eabef24c69bd602e72a9e57be29da367374ab153f62d538245b3a52 +size 205517 diff --git a/optimized/CALLHOME/ls_eend_ch_200ms.mlpackage/Manifest.json b/optimized/CALLHOME/ls_eend_ch_200ms.mlpackage/Manifest.json index 5aae696ac4244c100528d6ee54fa303ade4aed4a..e527d1c4d8001b0d0e8e8de63e64852c5a2f4d23 100644 --- a/optimized/CALLHOME/ls_eend_ch_200ms.mlpackage/Manifest.json +++ b/optimized/CALLHOME/ls_eend_ch_200ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "5054042F-ED8F-4239-83A8-5398612FFA7C": { - "author": "com.apple.CoreML", - "description": "CoreML Model Weights", - "name": "weights", - "path": "com.apple.CoreML/weights" - }, - "6078F0EC-C7FD-44A4-8321-2ED0EC18059F": { + "73DF81F2-B1E7-4461-BF6E-7A5F4ABBB449": { "author": "com.apple.CoreML", "description": "CoreML Model Specification", "name": "model.mlmodel", "path": "com.apple.CoreML/model.mlmodel" + }, + "9F478AAC-BA22-4C20-979A-B4FEC4B1EDF3": { + "author": "com.apple.CoreML", + "description": "CoreML Model Weights", + "name": "weights", + "path": "com.apple.CoreML/weights" } }, - "rootModelIdentifier": "6078F0EC-C7FD-44A4-8321-2ED0EC18059F" + "rootModelIdentifier": "73DF81F2-B1E7-4461-BF6E-7A5F4ABBB449" } diff --git a/optimized/CALLHOME/ls_eend_ch_300ms.mlmodelc/analytics/coremldata.bin b/optimized/CALLHOME/ls_eend_ch_300ms.mlmodelc/analytics/coremldata.bin index 3244ccc7ba4e6af92381c97e646272c30b27ed9d..577f50c023818b6e929edfff5e5fcebf9c3ec5c5 100644 --- a/optimized/CALLHOME/ls_eend_ch_300ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/CALLHOME/ls_eend_ch_300ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cd066903aeec6627ad4081368fa5c079a97cb566cfbe97925f3a553fa215bdc3 +oid sha256:04aa4f175ba19fb3db9d7af02a341f9d146c1cbba53da7fbfeb7ec555457a857 size 243 diff --git a/optimized/CALLHOME/ls_eend_ch_300ms.mlmodelc/coremldata.bin b/optimized/CALLHOME/ls_eend_ch_300ms.mlmodelc/coremldata.bin index 9bd184b8e5d9d95e42370f86cdbb301b0ca558ff..2d000cd605440772086dee81d548a948b2eeadd1 100644 --- a/optimized/CALLHOME/ls_eend_ch_300ms.mlmodelc/coremldata.bin +++ b/optimized/CALLHOME/ls_eend_ch_300ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0dbb2d3ce0638f74c9af56436430bd1fe4bd191ffa0edfbe386a47942ae8f020 -size 1259 +oid sha256:46e82b5928e1e10ecab14d475d39fd7299df7d7cbfae75fbf7fbae5e80e1b9f8 +size 1350 diff --git a/optimized/CALLHOME/ls_eend_ch_300ms.mlmodelc/metadata.json b/optimized/CALLHOME/ls_eend_ch_300ms.mlmodelc/metadata.json index 314b97c6a4a9ddcd364c4d325ab32a2e9286948a..5bfbed4b4f45eddf6143fedcaeeb4a5927b1f34d 100644 --- a/optimized/CALLHOME/ls_eend_ch_300ms.mlmodelc/metadata.json +++ b/optimized/CALLHOME/ls_eend_ch_300ms.mlmodelc/metadata.json @@ -96,30 +96,30 @@ "Ios17.reshape" : 92, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 69, - "Split" : 12, + "Ios17.transpose" : 57, + "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 82, "Tile" : 13, - "Ios17.sliceByIndex" : 76, - "Ios16.sigmoid" : 13, + "Ios17.sliceByIndex" : 60, + "Ios16.sigmoid" : 5, "Ios16.reduceSum" : 1, "Ios17.squeeze" : 2, - "Ios17.layerNorm" : 41, - "Ios17.batchNorm" : 12, + "Ios17.layerNorm" : 33, + "Ios17.batchNorm" : 4, "Ios17.cast" : 24, "Ios17.sqrt" : 18, "Ios17.sub" : 24, - "Ios17.conv" : 37, + "Ios17.conv" : 13, "Ios17.realDiv" : 20, "Ios17.linear" : 56, "Ios17.concat" : 18, "Ios17.clip" : 30, "Ios17.greaterEqual" : 12, - "Ios16.silu" : 26, + "Ios16.silu" : 18, "Stack" : 6, "Ios16.cumsum" : 1, - "Ios17.mul" : 103 + "Ios17.mul" : 95 }, "computePrecision" : "Mixed (Float32, Int32)", "isUpdatable" : "0", @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"ch\", \"model_label\": \"CALLHOME\", \"variant\": \"pipeline\", \"T\": 3, \"max_speakers\": 7, \"max_nspks\": 9, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"ch\", \"model_label\": \"CALLHOME\", \"variant\": \"pipeline\", \"chunk_size\": 3, \"step_duration_ms\": 300, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 7, \"max_nspks\": 9, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/CALLHOME/ls_eend_ch_300ms.mlmodelc/model.mil b/optimized/CALLHOME/ls_eend_ch_300ms.mlmodelc/model.mil index 3e3093e9df7d62e1d834d006116624ac2eab279a..315115211bcdd5f11ca6741e0fe77fbc1c0f84c7 100644 --- a/optimized/CALLHOME/ls_eend_ch_300ms.mlmodelc/model.mil +++ b/optimized/CALLHOME/ls_eend_ch_300ms.mlmodelc/model.mil @@ -217,6 +217,7 @@ program(1.0) tensor var_21 = const()[name = tensor("op_21"), val = tensor(0x0p+0)]; tensor var_24 = const()[name = tensor("op_24"), val = tensor(0x1.0c6f7ap-20)]; tensor var_27 = const()[name = tensor("op_27"), val = tensor(2)]; + tensor var_30 = const()[name = tensor("op_30"), val = tensor(0)]; tensor var_33 = const()[name = tensor("op_33"), val = tensor(1)]; tensor var_35 = const()[name = tensor("op_35"), val = tensor(0x1.4f8b58p-17)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(true)]; @@ -263,8 +264,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_69")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_70")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_196 = const()[name = tensor("op_196"), val = tensor([3, 1])]; tensor var_197 = reshape(shape = var_196, x = sqrt_s_t_1)[name = tensor("op_197")]; @@ -272,7 +273,7 @@ program(1.0) tensor var_199 = mul(x = qk_1, y = M_1)[name = tensor("op_199")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_68")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_199, y = v_1)[name = tensor("inner_1")]; tensor var_201_transpose_x_0 = const()[name = tensor("op_201_transpose_x_0"), val = tensor(false)]; tensor var_201_transpose_y_0 = const()[name = tensor("op_201_transpose_y_0"), val = tensor(false)]; @@ -293,7 +294,7 @@ program(1.0) tensor var_214 = real_div(x = new_kv_unnorm_1, y = var_213)[name = tensor("op_214")]; tensor var_215_perm_0 = const()[name = tensor("op_215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_67")]; + tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_24, x = var_215)[name = tensor("out_3")]; tensor var_219 = const()[name = tensor("op_219"), val = tensor([1, 3, 256])]; tensor out_5 = reshape(shape = var_219, x = out_3)[name = tensor("out_5")]; @@ -341,218 +342,142 @@ program(1.0) tensor var_252 = sub(x = var_17, y = use_roll_1)[name = tensor("op_252")]; tensor var_253 = mul(x = filled_window_1, y = var_252)[name = tensor("op_253")]; tensor var_254 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_254")]; - tensor input_21 = add(x = var_253, y = var_254)[name = tensor("input_21")]; + tensor window_3 = add(x = var_253, y = var_254)[name = tensor("window_3")]; tensor var_256 = const()[name = tensor("op_256"), val = tensor(0x1p+0)]; tensor var_257 = add(x = filled_1, y = var_256)[name = tensor("op_257")]; tensor clip_1 = clip(alpha = var_21, beta = var_16, x = var_257)[name = tensor("clip_1")]; + tensor var_260_begin_0 = const()[name = tensor("op_260_begin_0"), val = tensor([0, 1, 0])]; + tensor var_260_end_0 = const()[name = tensor("op_260_end_0"), val = tensor([1, 2, 256])]; + tensor var_260_end_mask_0 = const()[name = tensor("op_260_end_mask_0"), val = tensor([true, false, true])]; + tensor var_260 = slice_by_index(begin = var_260_begin_0, end = var_260_end_0, end_mask = var_260_end_mask_0, x = x_3)[name = tensor("op_260")]; + tensor var_263_begin_0 = const()[name = tensor("op_263_begin_0"), val = tensor([0, 1, 0])]; + tensor var_263_end_0 = const()[name = tensor("op_263_end_0"), val = tensor([1, 16, 256])]; + tensor var_263_end_mask_0 = const()[name = tensor("op_263_end_mask_0"), val = tensor([true, true, true])]; + tensor var_263 = slice_by_index(begin = var_263_begin_0, end = var_263_end_0, end_mask = var_263_end_mask_0, x = window_3)[name = tensor("op_263")]; + tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; + tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_263, var_260))[name = tensor("rolled_3")]; + tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; + tensor var_268 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_268")]; + tensor var_269_dtype_0 = const()[name = tensor("op_269_dtype_0"), val = tensor("fp32")]; + tensor var_270 = const()[name = tensor("op_270"), val = tensor([1, 16, 1])]; + tensor var_269 = cast(dtype = var_269_dtype_0, x = var_268)[name = tensor("cast_157")]; + tensor fill_mask_3 = reshape(shape = var_270, x = var_269)[name = tensor("fill_mask_3")]; + tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_260)[name = tensor("x_expanded_3")]; + tensor var_274 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_274")]; + tensor var_275 = mul(x = window_3, y = var_274)[name = tensor("op_275")]; + tensor var_276 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_276")]; + tensor filled_window_3 = add(x = var_275, y = var_276)[name = tensor("filled_window_3")]; + tensor var_278 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_278")]; + tensor var_279_dtype_0 = const()[name = tensor("op_279_dtype_0"), val = tensor("fp32")]; + tensor var_280 = const()[name = tensor("op_280"), val = tensor([1, 1, 1])]; + tensor var_279 = cast(dtype = var_279_dtype_0, x = var_278)[name = tensor("cast_156")]; + tensor use_roll_3 = reshape(shape = var_280, x = var_279)[name = tensor("use_roll_3")]; + tensor var_282 = sub(x = var_17, y = use_roll_3)[name = tensor("op_282")]; + tensor var_283 = mul(x = filled_window_3, y = var_282)[name = tensor("op_283")]; + tensor var_284 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_284")]; + tensor window_5 = add(x = var_283, y = var_284)[name = tensor("window_5")]; + tensor var_286 = const()[name = tensor("op_286"), val = tensor(0x1p+0)]; + tensor var_287 = add(x = clip_1, y = var_286)[name = tensor("op_287")]; + tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_287)[name = tensor("clip_3")]; + tensor var_290_begin_0 = const()[name = tensor("op_290_begin_0"), val = tensor([0, 2, 0])]; + tensor var_290_end_0 = const()[name = tensor("op_290_end_0"), val = tensor([1, 1, 256])]; + tensor var_290_end_mask_0 = const()[name = tensor("op_290_end_mask_0"), val = tensor([true, true, true])]; + tensor var_290 = slice_by_index(begin = var_290_begin_0, end = var_290_end_0, end_mask = var_290_end_mask_0, x = x_3)[name = tensor("op_290")]; + tensor var_293_begin_0 = const()[name = tensor("op_293_begin_0"), val = tensor([0, 1, 0])]; + tensor var_293_end_0 = const()[name = tensor("op_293_end_0"), val = tensor([1, 16, 256])]; + tensor var_293_end_mask_0 = const()[name = tensor("op_293_end_mask_0"), val = tensor([true, true, true])]; + tensor var_293 = slice_by_index(begin = var_293_begin_0, end = var_293_end_0, end_mask = var_293_end_mask_0, x = window_5)[name = tensor("op_293")]; + tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; + tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_293, var_290))[name = tensor("rolled_5")]; + tensor clip_4 = clip(alpha = var_21, beta = var_20, x = clip_3)[name = tensor("clip_4")]; + tensor var_298 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_298")]; + tensor var_299_dtype_0 = const()[name = tensor("op_299_dtype_0"), val = tensor("fp32")]; + tensor var_300 = const()[name = tensor("op_300"), val = tensor([1, 16, 1])]; + tensor var_299 = cast(dtype = var_299_dtype_0, x = var_298)[name = tensor("cast_155")]; + tensor fill_mask_5 = reshape(shape = var_300, x = var_299)[name = tensor("fill_mask_5")]; + tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_290)[name = tensor("x_expanded_5")]; + tensor var_304 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_304")]; + tensor var_305 = mul(x = window_5, y = var_304)[name = tensor("op_305")]; + tensor var_306 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_306")]; + tensor filled_window_5 = add(x = var_305, y = var_306)[name = tensor("filled_window_5")]; + tensor var_308 = greater_equal(x = clip_3, y = var_16)[name = tensor("op_308")]; + tensor var_309_dtype_0 = const()[name = tensor("op_309_dtype_0"), val = tensor("fp32")]; + tensor var_310 = const()[name = tensor("op_310"), val = tensor([1, 1, 1])]; + tensor var_309 = cast(dtype = var_309_dtype_0, x = var_308)[name = tensor("cast_154")]; + tensor use_roll_5 = reshape(shape = var_310, x = var_309)[name = tensor("use_roll_5")]; + tensor var_312 = sub(x = var_17, y = use_roll_5)[name = tensor("op_312")]; + tensor var_313 = mul(x = filled_window_5, y = var_312)[name = tensor("op_313")]; + tensor var_314 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_314")]; + tensor window_7 = add(x = var_313, y = var_314)[name = tensor("window_7")]; + tensor var_316 = const()[name = tensor("op_316"), val = tensor(0x1p+0)]; + tensor var_317 = add(x = clip_3, y = var_316)[name = tensor("op_317")]; + tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_317)[name = tensor("clip_5")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_30, interleave = input_21_interleave_0, values = (window_3, window_5, window_7))[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; - tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; + tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; tensor inputs_3_pad_type_0 = const()[name = tensor("inputs_3_pad_type_0"), val = tensor("valid")]; tensor inputs_3_strides_0 = const()[name = tensor("inputs_3_strides_0"), val = tensor([1])]; tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_66")]; - tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_278_split_sizes_0 = const()[name = tensor("op_278_split_sizes_0"), val = tensor([256, 256])]; - tensor var_278_axis_0 = const()[name = tensor("op_278_axis_0"), val = tensor(1)]; - tensor var_278_0, tensor var_278_1 = split(axis = var_278_axis_0, split_sizes = var_278_split_sizes_0, x = inputs_3)[name = tensor("op_278")]; - tensor var_280 = sigmoid(x = var_278_1)[name = tensor("op_280")]; - tensor inputs_5 = mul(x = var_278_0, y = var_280)[name = tensor("inputs_5")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; + tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; + tensor var_340_split_sizes_0 = const()[name = tensor("op_340_split_sizes_0"), val = tensor([256, 256])]; + tensor var_340_axis_0 = const()[name = tensor("op_340_axis_0"), val = tensor(1)]; + tensor var_340_0, tensor var_340_1 = split(axis = var_340_axis_0, split_sizes = var_340_split_sizes_0, x = inputs_3)[name = tensor("op_340")]; + tensor var_342 = sigmoid(x = var_340_1)[name = tensor("op_342")]; + tensor inputs_5 = mul(x = var_340_0, y = var_342)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; tensor outputs_aug_1_strides_0 = const()[name = tensor("outputs_aug_1_strides_0"), val = tensor([1])]; tensor outputs_aug_1_dilations_0 = const()[name = tensor("outputs_aug_1_dilations_0"), val = tensor([1])]; - tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; + tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; tensor input_25_begin_0 = const()[name = tensor("input_25_begin_0"), val = tensor([0, 0, 0])]; - tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([1, 256, 16])]; + tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([3, 256, 16])]; tensor input_25_end_mask_0 = const()[name = tensor("input_25_end_mask_0"), val = tensor([true, true, false])]; - tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; - tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; - tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; + tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; + tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; + tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; tensor input_29_pad_type_0 = const()[name = tensor("input_29_pad_type_0"), val = tensor("valid")]; tensor input_29_strides_0 = const()[name = tensor("input_29_strides_0"), val = tensor([1])]; tensor input_29_pad_0 = const()[name = tensor("input_29_pad_0"), val = tensor([0, 0])]; tensor input_29_dilations_0 = const()[name = tensor("input_29_dilations_0"), val = tensor([1])]; tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; - tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; + tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_311_begin_0 = const()[name = tensor("op_311_begin_0"), val = tensor([0, -1, 0])]; - tensor var_311_end_0 = const()[name = tensor("op_311_end_0"), val = tensor([1, 16, 256])]; - tensor var_311_end_mask_0 = const()[name = tensor("op_311_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_65")]; - tensor var_311 = slice_by_index(begin = var_311_begin_0, end = var_311_end_0, end_mask = var_311_end_mask_0, x = conv_out_1)[name = tensor("op_311")]; - tensor var_314_begin_0 = const()[name = tensor("op_314_begin_0"), val = tensor([0, 1, 0])]; - tensor var_314_end_0 = const()[name = tensor("op_314_end_0"), val = tensor([1, 2, 256])]; - tensor var_314_end_mask_0 = const()[name = tensor("op_314_end_mask_0"), val = tensor([true, false, true])]; - tensor var_314 = slice_by_index(begin = var_314_begin_0, end = var_314_end_0, end_mask = var_314_end_mask_0, x = x_3)[name = tensor("op_314")]; - tensor var_317_begin_0 = const()[name = tensor("op_317_begin_0"), val = tensor([0, 1, 0])]; - tensor var_317_end_0 = const()[name = tensor("op_317_end_0"), val = tensor([1, 16, 256])]; - tensor var_317_end_mask_0 = const()[name = tensor("op_317_end_mask_0"), val = tensor([true, true, true])]; - tensor var_317 = slice_by_index(begin = var_317_begin_0, end = var_317_end_0, end_mask = var_317_end_mask_0, x = input_21)[name = tensor("op_317")]; - tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_317, var_314))[name = tensor("rolled_3")]; - tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; - tensor var_322 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_322")]; - tensor var_323_dtype_0 = const()[name = tensor("op_323_dtype_0"), val = tensor("fp32")]; - tensor var_324 = const()[name = tensor("op_324"), val = tensor([1, 16, 1])]; - tensor var_323 = cast(dtype = var_323_dtype_0, x = var_322)[name = tensor("cast_157")]; - tensor fill_mask_3 = reshape(shape = var_324, x = var_323)[name = tensor("fill_mask_3")]; - tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_314)[name = tensor("x_expanded_3")]; - tensor var_328 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_328")]; - tensor var_329 = mul(x = input_21, y = var_328)[name = tensor("op_329")]; - tensor var_330 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_330")]; - tensor filled_window_3 = add(x = var_329, y = var_330)[name = tensor("filled_window_3")]; - tensor var_332 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_332")]; - tensor var_333_dtype_0 = const()[name = tensor("op_333_dtype_0"), val = tensor("fp32")]; - tensor var_334 = const()[name = tensor("op_334"), val = tensor([1, 1, 1])]; - tensor var_333 = cast(dtype = var_333_dtype_0, x = var_332)[name = tensor("cast_156")]; - tensor use_roll_3 = reshape(shape = var_334, x = var_333)[name = tensor("use_roll_3")]; - tensor var_336 = sub(x = var_17, y = use_roll_3)[name = tensor("op_336")]; - tensor var_337 = mul(x = filled_window_3, y = var_336)[name = tensor("op_337")]; - tensor var_338 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_338")]; - tensor input_31 = add(x = var_337, y = var_338)[name = tensor("input_31")]; - tensor var_340 = const()[name = tensor("op_340"), val = tensor(0x1p+0)]; - tensor var_341 = add(x = clip_1, y = var_340)[name = tensor("op_341")]; - tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_341)[name = tensor("clip_3")]; + tensor var_373_begin_0 = const()[name = tensor("op_373_begin_0"), val = tensor([0, -1, 0])]; + tensor var_373_end_0 = const()[name = tensor("op_373_end_0"), val = tensor([3, 16, 256])]; + tensor var_373_end_mask_0 = const()[name = tensor("op_373_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_373 = slice_by_index(begin = var_373_begin_0, end = var_373_end_0, end_mask = var_373_end_mask_0, x = conv_out_1)[name = tensor("op_373")]; + tensor var_375_perm_0 = const()[name = tensor("op_375_perm_0"), val = tensor([1, 0, 2])]; + tensor var_375 = transpose(perm = var_375_perm_0, x = var_373)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_375)[name = tensor("input_31")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; + tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; + tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; + tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; + tensor var_398 = const()[name = tensor("op_398"), val = tensor(0x1p-1)]; + tensor var_399 = mul(x = input_39, y = var_398)[name = tensor("op_399")]; + tensor input_41 = add(x = var_399, y = input_31)[name = tensor("input_41")]; + tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; + tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; + tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; + tensor input_45 = layer_norm(axes = input_45_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_43)[name = tensor("input_45")]; + tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; + tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; + tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; + tensor var_428 = const()[name = tensor("op_428"), val = tensor(0x1p-1)]; + tensor var_429 = mul(x = input_51, y = var_428)[name = tensor("op_429")]; + tensor input_53 = add(x = var_429, y = input_43)[name = tensor("input_53")]; tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; - tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_31)[name = tensor("x_7")]; - tensor input_33_perm_0 = const()[name = tensor("input_33_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_9_pad_type_0 = const()[name = tensor("inputs_9_pad_type_0"), val = tensor("valid")]; - tensor inputs_9_strides_0 = const()[name = tensor("inputs_9_strides_0"), val = tensor([1])]; - tensor inputs_9_pad_0 = const()[name = tensor("inputs_9_pad_0"), val = tensor([0, 0])]; - tensor inputs_9_dilations_0 = const()[name = tensor("inputs_9_dilations_0"), val = tensor([1])]; - tensor inputs_9_groups_0 = const()[name = tensor("inputs_9_groups_0"), val = tensor(1)]; - tensor input_33 = transpose(perm = input_33_perm_0, x = x_7)[name = tensor("transpose_64")]; - tensor inputs_9 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_9_dilations_0, groups = inputs_9_groups_0, pad = inputs_9_pad_0, pad_type = inputs_9_pad_type_0, strides = inputs_9_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_33)[name = tensor("inputs_9")]; - tensor var_362_split_sizes_0 = const()[name = tensor("op_362_split_sizes_0"), val = tensor([256, 256])]; - tensor var_362_axis_0 = const()[name = tensor("op_362_axis_0"), val = tensor(1)]; - tensor var_362_0, tensor var_362_1 = split(axis = var_362_axis_0, split_sizes = var_362_split_sizes_0, x = inputs_9)[name = tensor("op_362")]; - tensor var_364 = sigmoid(x = var_362_1)[name = tensor("op_364")]; - tensor inputs_11 = mul(x = var_362_0, y = var_364)[name = tensor("inputs_11")]; - tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; - tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; - tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; - tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_11)[name = tensor("outputs_aug_3")]; - tensor input_35_begin_0 = const()[name = tensor("input_35_begin_0"), val = tensor([0, 0, 0])]; - tensor input_35_end_0 = const()[name = tensor("input_35_end_0"), val = tensor([1, 256, 16])]; - tensor input_35_end_mask_0 = const()[name = tensor("input_35_end_mask_0"), val = tensor([true, true, false])]; - tensor input_35 = slice_by_index(begin = input_35_begin_0, end = input_35_end_0, end_mask = input_35_end_mask_0, x = outputs_aug_3)[name = tensor("input_35")]; - tensor inputs_13 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_35)[name = tensor("inputs_13")]; - tensor input_37 = silu(x = inputs_13)[name = tensor("input_37")]; - tensor input_39_pad_type_0 = const()[name = tensor("input_39_pad_type_0"), val = tensor("valid")]; - tensor input_39_strides_0 = const()[name = tensor("input_39_strides_0"), val = tensor([1])]; - tensor input_39_pad_0 = const()[name = tensor("input_39_pad_0"), val = tensor([0, 0])]; - tensor input_39_dilations_0 = const()[name = tensor("input_39_dilations_0"), val = tensor([1])]; - tensor input_39_groups_0 = const()[name = tensor("input_39_groups_0"), val = tensor(1)]; - tensor input_39 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_37)[name = tensor("input_39")]; - tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_395_begin_0 = const()[name = tensor("op_395_begin_0"), val = tensor([0, -1, 0])]; - tensor var_395_end_0 = const()[name = tensor("op_395_end_0"), val = tensor([1, 16, 256])]; - tensor var_395_end_mask_0 = const()[name = tensor("op_395_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_39)[name = tensor("transpose_63")]; - tensor var_395 = slice_by_index(begin = var_395_begin_0, end = var_395_end_0, end_mask = var_395_end_mask_0, x = conv_out_3)[name = tensor("op_395")]; - tensor var_398_begin_0 = const()[name = tensor("op_398_begin_0"), val = tensor([0, 2, 0])]; - tensor var_398_end_0 = const()[name = tensor("op_398_end_0"), val = tensor([1, 1, 256])]; - tensor var_398_end_mask_0 = const()[name = tensor("op_398_end_mask_0"), val = tensor([true, true, true])]; - tensor var_398 = slice_by_index(begin = var_398_begin_0, end = var_398_end_0, end_mask = var_398_end_mask_0, x = x_3)[name = tensor("op_398")]; - tensor var_401_begin_0 = const()[name = tensor("op_401_begin_0"), val = tensor([0, 1, 0])]; - tensor var_401_end_0 = const()[name = tensor("op_401_end_0"), val = tensor([1, 16, 256])]; - tensor var_401_end_mask_0 = const()[name = tensor("op_401_end_mask_0"), val = tensor([true, true, true])]; - tensor var_401 = slice_by_index(begin = var_401_begin_0, end = var_401_end_0, end_mask = var_401_end_mask_0, x = input_31)[name = tensor("op_401")]; - tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_401, var_398))[name = tensor("rolled_5")]; - tensor clip_4 = clip(alpha = var_21, beta = var_20, x = clip_3)[name = tensor("clip_4")]; - tensor var_406 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_406")]; - tensor var_407_dtype_0 = const()[name = tensor("op_407_dtype_0"), val = tensor("fp32")]; - tensor var_408 = const()[name = tensor("op_408"), val = tensor([1, 16, 1])]; - tensor var_407 = cast(dtype = var_407_dtype_0, x = var_406)[name = tensor("cast_155")]; - tensor fill_mask_5 = reshape(shape = var_408, x = var_407)[name = tensor("fill_mask_5")]; - tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_398)[name = tensor("x_expanded_5")]; - tensor var_412 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_412")]; - tensor var_413 = mul(x = input_31, y = var_412)[name = tensor("op_413")]; - tensor var_414 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_414")]; - tensor filled_window_5 = add(x = var_413, y = var_414)[name = tensor("filled_window_5")]; - tensor var_416 = greater_equal(x = clip_3, y = var_16)[name = tensor("op_416")]; - tensor var_417_dtype_0 = const()[name = tensor("op_417_dtype_0"), val = tensor("fp32")]; - tensor var_418 = const()[name = tensor("op_418"), val = tensor([1, 1, 1])]; - tensor var_417 = cast(dtype = var_417_dtype_0, x = var_416)[name = tensor("cast_154")]; - tensor use_roll_5 = reshape(shape = var_418, x = var_417)[name = tensor("use_roll_5")]; - tensor var_420 = sub(x = var_17, y = use_roll_5)[name = tensor("op_420")]; - tensor var_421 = mul(x = filled_window_5, y = var_420)[name = tensor("op_421")]; - tensor var_422 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_422")]; - tensor input_41 = add(x = var_421, y = var_422)[name = tensor("input_41")]; - tensor var_424 = const()[name = tensor("op_424"), val = tensor(0x1p+0)]; - tensor var_425 = add(x = clip_3, y = var_424)[name = tensor("op_425")]; - tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_425)[name = tensor("clip_5")]; - tensor x_9_axes_0 = const()[name = tensor("x_9_axes_0"), val = tensor([-1])]; - tensor x_9 = layer_norm(axes = x_9_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_41)[name = tensor("x_9")]; - tensor input_43_perm_0 = const()[name = tensor("input_43_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_15_pad_type_0 = const()[name = tensor("inputs_15_pad_type_0"), val = tensor("valid")]; - tensor inputs_15_strides_0 = const()[name = tensor("inputs_15_strides_0"), val = tensor([1])]; - tensor inputs_15_pad_0 = const()[name = tensor("inputs_15_pad_0"), val = tensor([0, 0])]; - tensor inputs_15_dilations_0 = const()[name = tensor("inputs_15_dilations_0"), val = tensor([1])]; - tensor inputs_15_groups_0 = const()[name = tensor("inputs_15_groups_0"), val = tensor(1)]; - tensor input_43 = transpose(perm = input_43_perm_0, x = x_9)[name = tensor("transpose_62")]; - tensor inputs_15 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_15_dilations_0, groups = inputs_15_groups_0, pad = inputs_15_pad_0, pad_type = inputs_15_pad_type_0, strides = inputs_15_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_43)[name = tensor("inputs_15")]; - tensor var_446_split_sizes_0 = const()[name = tensor("op_446_split_sizes_0"), val = tensor([256, 256])]; - tensor var_446_axis_0 = const()[name = tensor("op_446_axis_0"), val = tensor(1)]; - tensor var_446_0, tensor var_446_1 = split(axis = var_446_axis_0, split_sizes = var_446_split_sizes_0, x = inputs_15)[name = tensor("op_446")]; - tensor var_448 = sigmoid(x = var_446_1)[name = tensor("op_448")]; - tensor inputs_17 = mul(x = var_446_0, y = var_448)[name = tensor("inputs_17")]; - tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; - tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; - tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; - tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_17)[name = tensor("outputs_aug_5")]; - tensor input_45_begin_0 = const()[name = tensor("input_45_begin_0"), val = tensor([0, 0, 0])]; - tensor input_45_end_0 = const()[name = tensor("input_45_end_0"), val = tensor([1, 256, 16])]; - tensor input_45_end_mask_0 = const()[name = tensor("input_45_end_mask_0"), val = tensor([true, true, false])]; - tensor input_45 = slice_by_index(begin = input_45_begin_0, end = input_45_end_0, end_mask = input_45_end_mask_0, x = outputs_aug_5)[name = tensor("input_45")]; - tensor inputs_19 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_45)[name = tensor("inputs_19")]; - tensor input_47 = silu(x = inputs_19)[name = tensor("input_47")]; - tensor input_49_pad_type_0 = const()[name = tensor("input_49_pad_type_0"), val = tensor("valid")]; - tensor input_49_strides_0 = const()[name = tensor("input_49_strides_0"), val = tensor([1])]; - tensor input_49_pad_0 = const()[name = tensor("input_49_pad_0"), val = tensor([0, 0])]; - tensor input_49_dilations_0 = const()[name = tensor("input_49_dilations_0"), val = tensor([1])]; - tensor input_49_groups_0 = const()[name = tensor("input_49_groups_0"), val = tensor(1)]; - tensor input_49 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_49_dilations_0, groups = input_49_groups_0, pad = input_49_pad_0, pad_type = input_49_pad_type_0, strides = input_49_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_47)[name = tensor("input_49")]; - tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_479_begin_0 = const()[name = tensor("op_479_begin_0"), val = tensor([0, -1, 0])]; - tensor var_479_end_0 = const()[name = tensor("op_479_end_0"), val = tensor([1, 16, 256])]; - tensor var_479_end_mask_0 = const()[name = tensor("op_479_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_49)[name = tensor("transpose_61")]; - tensor var_479 = slice_by_index(begin = var_479_begin_0, end = var_479_end_0, end_mask = var_479_end_mask_0, x = conv_out_5)[name = tensor("op_479")]; - tensor var_482_interleave_0 = const()[name = tensor("op_482_interleave_0"), val = tensor(false)]; - tensor var_482 = concat(axis = var_33, interleave = var_482_interleave_0, values = (var_311, var_395, var_479))[name = tensor("op_482")]; - tensor input_51 = add(x = x_3, y = var_482)[name = tensor("input_51")]; - tensor input_53_axes_0 = const()[name = tensor("input_53_axes_0"), val = tensor([-1])]; - tensor input_53 = layer_norm(axes = input_53_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_51)[name = tensor("input_53")]; - tensor inputs_21 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_53)[name = tensor("linear_8")]; - tensor input_55 = silu(x = inputs_21)[name = tensor("input_55")]; - tensor input_59 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_55)[name = tensor("linear_9")]; - tensor var_505 = const()[name = tensor("op_505"), val = tensor(0x1p-1)]; - tensor var_506 = mul(x = input_59, y = var_505)[name = tensor("op_506")]; - tensor input_61 = add(x = var_506, y = input_51)[name = tensor("input_61")]; - tensor input_63_axes_0 = const()[name = tensor("input_63_axes_0"), val = tensor([-1])]; - tensor input_63 = layer_norm(axes = input_63_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_61)[name = tensor("input_63")]; - tensor input_65_axes_0 = const()[name = tensor("input_65_axes_0"), val = tensor([-1])]; - tensor input_65 = layer_norm(axes = input_65_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_63)[name = tensor("input_65")]; - tensor inputs_23 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_65)[name = tensor("linear_10")]; - tensor input_67 = silu(x = inputs_23)[name = tensor("input_67")]; - tensor input_71 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_67)[name = tensor("linear_11")]; - tensor var_535 = const()[name = tensor("op_535"), val = tensor(0x1p-1)]; - tensor var_536 = mul(x = input_71, y = var_535)[name = tensor("op_536")]; - tensor input_73 = add(x = var_536, y = input_63)[name = tensor("input_73")]; - tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; - tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_73)[name = tensor("x_11")]; + tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_3_end_0 = const()[name = tensor("prev_kv_3_end_0"), val = tensor([2, 1, 4, 64, 64])]; tensor prev_kv_3_end_mask_0 = const()[name = tensor("prev_kv_3_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -563,316 +488,240 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_550 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_11)[name = tensor("linear_12")]; - tensor var_551 = const()[name = tensor("op_551"), val = tensor([1, 3, 4, 64])]; - tensor var_552 = reshape(shape = var_551, x = var_550)[name = tensor("op_552")]; + tensor var_443 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_444 = const()[name = tensor("op_444"), val = tensor([1, 3, 4, 64])]; + tensor var_445 = reshape(shape = var_444, x = var_443)[name = tensor("op_445")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_556 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_11)[name = tensor("linear_13")]; - tensor var_557 = const()[name = tensor("op_557"), val = tensor(0x1p-3)]; - tensor var_558 = mul(x = var_556, y = var_557)[name = tensor("op_558")]; - tensor var_559 = const()[name = tensor("op_559"), val = tensor([1, 3, 4, 64])]; - tensor var_560 = reshape(shape = var_559, x = var_558)[name = tensor("op_560")]; + tensor var_449 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_450 = const()[name = tensor("op_450"), val = tensor(0x1p-3)]; + tensor var_451 = mul(x = var_449, y = var_450)[name = tensor("op_451")]; + tensor var_452 = const()[name = tensor("op_452"), val = tensor([1, 3, 4, 64])]; + tensor var_453 = reshape(shape = var_452, x = var_451)[name = tensor("op_453")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_564 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_11)[name = tensor("linear_14")]; - tensor var_565 = const()[name = tensor("op_565"), val = tensor([1, 3, 4, 64])]; - tensor var_566 = reshape(shape = var_565, x = var_564)[name = tensor("op_566")]; + tensor var_457 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_458 = const()[name = tensor("op_458"), val = tensor([1, 3, 4, 64])]; + tensor var_459 = reshape(shape = var_458, x = var_457)[name = tensor("op_459")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_77 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_11)[name = tensor("linear_15")]; + tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; tensor s_t_3 = add(x = prev_scale_3, y = encoder__t_index)[name = tensor("s_t_3")]; tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_560)[name = tensor("transpose_59")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_552)[name = tensor("transpose_60")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_453)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_445)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_576 = const()[name = tensor("op_576"), val = tensor([3, 1])]; - tensor var_577 = reshape(shape = var_576, x = sqrt_s_t_3)[name = tensor("op_577")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_577)[name = tensor("M_3")]; - tensor var_579 = mul(x = qk_3, y = M_3)[name = tensor("op_579")]; + tensor var_469 = const()[name = tensor("op_469"), val = tensor([3, 1])]; + tensor var_470 = reshape(shape = var_469, x = sqrt_s_t_3)[name = tensor("op_470")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_470)[name = tensor("M_3")]; + tensor var_472 = mul(x = qk_3, y = M_3)[name = tensor("op_472")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_566)[name = tensor("transpose_58")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_579, y = v_3)[name = tensor("inner_3")]; - tensor var_581_transpose_x_0 = const()[name = tensor("op_581_transpose_x_0"), val = tensor(false)]; - tensor var_581_transpose_y_0 = const()[name = tensor("op_581_transpose_y_0"), val = tensor(false)]; - tensor var_581 = matmul(transpose_x = var_581_transpose_x_0, transpose_y = var_581_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_581")]; - tensor var_582 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_582")]; - tensor var_583 = const()[name = tensor("op_583"), val = tensor([1, 1, 3, 1])]; - tensor var_584 = reshape(shape = var_583, x = var_582)[name = tensor("op_584")]; - tensor cross_3 = mul(x = var_581, y = var_584)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_459)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_472, y = v_3)[name = tensor("inner_3")]; + tensor var_474_transpose_x_0 = const()[name = tensor("op_474_transpose_x_0"), val = tensor(false)]; + tensor var_474_transpose_y_0 = const()[name = tensor("op_474_transpose_y_0"), val = tensor(false)]; + tensor var_474 = matmul(transpose_x = var_474_transpose_x_0, transpose_y = var_474_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_474")]; + tensor var_475 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_475")]; + tensor var_476 = const()[name = tensor("op_476"), val = tensor([1, 1, 3, 1])]; + tensor var_477 = reshape(shape = var_476, x = var_475)[name = tensor("op_477")]; + tensor cross_3 = mul(x = var_474, y = var_477)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_587 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_587")]; - tensor var_589_transpose_x_1 = const()[name = tensor("op_589_transpose_x_1"), val = tensor(true)]; - tensor var_589_transpose_y_1 = const()[name = tensor("op_589_transpose_y_1"), val = tensor(false)]; - tensor var_589 = matmul(transpose_x = var_589_transpose_x_1, transpose_y = var_589_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_589")]; - tensor new_kv_unnorm_3 = add(x = var_587, y = var_589)[name = tensor("new_kv_unnorm_3")]; - tensor var_591 = const()[name = tensor("op_591"), val = tensor(0x1.8p+1)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_591)[name = tensor("new_scale_3")]; - tensor var_593 = sqrt(x = new_scale_3)[name = tensor("op_593")]; - tensor var_594 = real_div(x = new_kv_unnorm_3, y = var_593)[name = tensor("op_594")]; - tensor var_595_perm_0 = const()[name = tensor("op_595_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_480 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_480")]; + tensor var_482_transpose_x_1 = const()[name = tensor("op_482_transpose_x_1"), val = tensor(true)]; + tensor var_482_transpose_y_1 = const()[name = tensor("op_482_transpose_y_1"), val = tensor(false)]; + tensor var_482 = matmul(transpose_x = var_482_transpose_x_1, transpose_y = var_482_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_482")]; + tensor new_kv_unnorm_3 = add(x = var_480, y = var_482)[name = tensor("new_kv_unnorm_3")]; + tensor var_484 = const()[name = tensor("op_484"), val = tensor(0x1.8p+1)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_484)[name = tensor("new_scale_3")]; + tensor var_486 = sqrt(x = new_scale_3)[name = tensor("op_486")]; + tensor var_487 = real_div(x = new_kv_unnorm_3, y = var_486)[name = tensor("op_487")]; + tensor var_488_perm_0 = const()[name = tensor("op_488_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_595 = transpose(perm = var_595_perm_0, x = out_7)[name = tensor("transpose_57")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_595)[name = tensor("out_9")]; - tensor var_599 = const()[name = tensor("op_599"), val = tensor([1, 3, 256])]; - tensor out_11 = reshape(shape = var_599, x = out_9)[name = tensor("out_11")]; - tensor var_601 = silu(x = input_77)[name = tensor("op_601")]; - tensor input_79 = mul(x = var_601, y = out_11)[name = tensor("input_79")]; - tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_79)[name = tensor("linear_16")]; - tensor x_13 = add(x = input_73, y = ret_out_3)[name = tensor("x_13")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor var_488 = transpose(perm = var_488_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_488)[name = tensor("out_9")]; + tensor var_492 = const()[name = tensor("op_492"), val = tensor([1, 3, 256])]; + tensor out_11 = reshape(shape = var_492, x = out_9)[name = tensor("out_11")]; + tensor var_494 = silu(x = input_57)[name = tensor("op_494")]; + tensor input_59 = mul(x = var_494, y = out_11)[name = tensor("input_59")]; + tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; + tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; + tensor window_9_begin_0 = const()[name = tensor("window_9_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_9_end_0 = const()[name = tensor("window_9_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_9_end_mask_0 = const()[name = tensor("window_9_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_9_squeeze_mask_0 = const()[name = tensor("window_9_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_9 = slice_by_index(begin = window_9_begin_0, end = window_9_end_0, end_mask = window_9_end_mask_0, squeeze_mask = window_9_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_9")]; tensor filled_7_begin_0 = const()[name = tensor("filled_7_begin_0"), val = tensor([1, 0])]; tensor filled_7_end_0 = const()[name = tensor("filled_7_end_0"), val = tensor([2, 1])]; tensor filled_7_end_mask_0 = const()[name = tensor("filled_7_end_mask_0"), val = tensor([false, true])]; tensor filled_7_squeeze_mask_0 = const()[name = tensor("filled_7_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_7 = slice_by_index(begin = filled_7_begin_0, end = filled_7_end_0, end_mask = filled_7_end_mask_0, squeeze_mask = filled_7_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_7")]; - tensor var_610_begin_0 = const()[name = tensor("op_610_begin_0"), val = tensor([0, 0, 0])]; - tensor var_610_end_0 = const()[name = tensor("op_610_end_0"), val = tensor([1, 1, 256])]; - tensor var_610_end_mask_0 = const()[name = tensor("op_610_end_mask_0"), val = tensor([true, false, true])]; - tensor var_610 = slice_by_index(begin = var_610_begin_0, end = var_610_end_0, end_mask = var_610_end_mask_0, x = x_13)[name = tensor("op_610")]; - tensor var_613_begin_0 = const()[name = tensor("op_613_begin_0"), val = tensor([0, 1, 0])]; - tensor var_613_end_0 = const()[name = tensor("op_613_end_0"), val = tensor([1, 16, 256])]; - tensor var_613_end_mask_0 = const()[name = tensor("op_613_end_mask_0"), val = tensor([true, true, true])]; - tensor var_613 = slice_by_index(begin = var_613_begin_0, end = var_613_end_0, end_mask = var_613_end_mask_0, x = window_3)[name = tensor("op_613")]; + tensor var_503_begin_0 = const()[name = tensor("op_503_begin_0"), val = tensor([0, 0, 0])]; + tensor var_503_end_0 = const()[name = tensor("op_503_end_0"), val = tensor([1, 1, 256])]; + tensor var_503_end_mask_0 = const()[name = tensor("op_503_end_mask_0"), val = tensor([true, false, true])]; + tensor var_503 = slice_by_index(begin = var_503_begin_0, end = var_503_end_0, end_mask = var_503_end_mask_0, x = x_9)[name = tensor("op_503")]; + tensor var_506_begin_0 = const()[name = tensor("op_506_begin_0"), val = tensor([0, 1, 0])]; + tensor var_506_end_0 = const()[name = tensor("op_506_end_0"), val = tensor([1, 16, 256])]; + tensor var_506_end_mask_0 = const()[name = tensor("op_506_end_mask_0"), val = tensor([true, true, true])]; + tensor var_506 = slice_by_index(begin = var_506_begin_0, end = var_506_end_0, end_mask = var_506_end_mask_0, x = window_9)[name = tensor("op_506")]; tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; - tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_613, var_610))[name = tensor("rolled_7")]; + tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_506, var_503))[name = tensor("rolled_7")]; tensor clip_6 = clip(alpha = var_21, beta = var_20, x = filled_7)[name = tensor("clip_6")]; - tensor var_618 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_618")]; - tensor var_619_dtype_0 = const()[name = tensor("op_619_dtype_0"), val = tensor("fp32")]; - tensor var_620 = const()[name = tensor("op_620"), val = tensor([1, 16, 1])]; - tensor var_619 = cast(dtype = var_619_dtype_0, x = var_618)[name = tensor("cast_153")]; - tensor fill_mask_7 = reshape(shape = var_620, x = var_619)[name = tensor("fill_mask_7")]; + tensor var_511 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_511")]; + tensor var_512_dtype_0 = const()[name = tensor("op_512_dtype_0"), val = tensor("fp32")]; + tensor var_513 = const()[name = tensor("op_513"), val = tensor([1, 16, 1])]; + tensor var_512 = cast(dtype = var_512_dtype_0, x = var_511)[name = tensor("cast_153")]; + tensor fill_mask_7 = reshape(shape = var_513, x = var_512)[name = tensor("fill_mask_7")]; tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_610)[name = tensor("x_expanded_7")]; - tensor var_624 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_624")]; - tensor var_625 = mul(x = window_3, y = var_624)[name = tensor("op_625")]; - tensor var_626 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_626")]; - tensor filled_window_7 = add(x = var_625, y = var_626)[name = tensor("filled_window_7")]; - tensor var_628 = greater_equal(x = filled_7, y = var_16)[name = tensor("op_628")]; - tensor var_629_dtype_0 = const()[name = tensor("op_629_dtype_0"), val = tensor("fp32")]; - tensor var_630 = const()[name = tensor("op_630"), val = tensor([1, 1, 1])]; - tensor var_629 = cast(dtype = var_629_dtype_0, x = var_628)[name = tensor("cast_152")]; - tensor use_roll_7 = reshape(shape = var_630, x = var_629)[name = tensor("use_roll_7")]; - tensor var_632 = sub(x = var_17, y = use_roll_7)[name = tensor("op_632")]; - tensor var_633 = mul(x = filled_window_7, y = var_632)[name = tensor("op_633")]; - tensor var_634 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_634")]; - tensor input_81 = add(x = var_633, y = var_634)[name = tensor("input_81")]; - tensor var_636 = const()[name = tensor("op_636"), val = tensor(0x1p+0)]; - tensor var_637 = add(x = filled_7, y = var_636)[name = tensor("op_637")]; - tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_637)[name = tensor("clip_7")]; - tensor x_15_axes_0 = const()[name = tensor("x_15_axes_0"), val = tensor([-1])]; - tensor x_15 = layer_norm(axes = x_15_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_81)[name = tensor("x_15")]; - tensor input_83_perm_0 = const()[name = tensor("input_83_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_25_pad_type_0 = const()[name = tensor("inputs_25_pad_type_0"), val = tensor("valid")]; - tensor inputs_25_strides_0 = const()[name = tensor("inputs_25_strides_0"), val = tensor([1])]; - tensor inputs_25_pad_0 = const()[name = tensor("inputs_25_pad_0"), val = tensor([0, 0])]; - tensor inputs_25_dilations_0 = const()[name = tensor("inputs_25_dilations_0"), val = tensor([1])]; - tensor inputs_25_groups_0 = const()[name = tensor("inputs_25_groups_0"), val = tensor(1)]; - tensor input_83 = transpose(perm = input_83_perm_0, x = x_15)[name = tensor("transpose_56")]; - tensor inputs_25 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_25_dilations_0, groups = inputs_25_groups_0, pad = inputs_25_pad_0, pad_type = inputs_25_pad_type_0, strides = inputs_25_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_83)[name = tensor("inputs_25")]; - tensor var_658_split_sizes_0 = const()[name = tensor("op_658_split_sizes_0"), val = tensor([256, 256])]; - tensor var_658_axis_0 = const()[name = tensor("op_658_axis_0"), val = tensor(1)]; - tensor var_658_0, tensor var_658_1 = split(axis = var_658_axis_0, split_sizes = var_658_split_sizes_0, x = inputs_25)[name = tensor("op_658")]; - tensor var_660 = sigmoid(x = var_658_1)[name = tensor("op_660")]; - tensor inputs_27 = mul(x = var_658_0, y = var_660)[name = tensor("inputs_27")]; - tensor outputs_aug_7_pad_type_0 = const()[name = tensor("outputs_aug_7_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_7_pad_0 = const()[name = tensor("outputs_aug_7_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_7_groups_0 = const()[name = tensor("outputs_aug_7_groups_0"), val = tensor(256)]; - tensor outputs_aug_7_strides_0 = const()[name = tensor("outputs_aug_7_strides_0"), val = tensor([1])]; - tensor outputs_aug_7_dilations_0 = const()[name = tensor("outputs_aug_7_dilations_0"), val = tensor([1])]; - tensor outputs_aug_7 = conv(dilations = outputs_aug_7_dilations_0, groups = outputs_aug_7_groups_0, pad = outputs_aug_7_pad_0, pad_type = outputs_aug_7_pad_type_0, strides = outputs_aug_7_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_27)[name = tensor("outputs_aug_7")]; - tensor input_85_begin_0 = const()[name = tensor("input_85_begin_0"), val = tensor([0, 0, 0])]; - tensor input_85_end_0 = const()[name = tensor("input_85_end_0"), val = tensor([1, 256, 16])]; - tensor input_85_end_mask_0 = const()[name = tensor("input_85_end_mask_0"), val = tensor([true, true, false])]; - tensor input_85 = slice_by_index(begin = input_85_begin_0, end = input_85_end_0, end_mask = input_85_end_mask_0, x = outputs_aug_7)[name = tensor("input_85")]; - tensor inputs_29 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_85)[name = tensor("inputs_29")]; - tensor input_87 = silu(x = inputs_29)[name = tensor("input_87")]; - tensor input_89_pad_type_0 = const()[name = tensor("input_89_pad_type_0"), val = tensor("valid")]; - tensor input_89_strides_0 = const()[name = tensor("input_89_strides_0"), val = tensor([1])]; - tensor input_89_pad_0 = const()[name = tensor("input_89_pad_0"), val = tensor([0, 0])]; - tensor input_89_dilations_0 = const()[name = tensor("input_89_dilations_0"), val = tensor([1])]; - tensor input_89_groups_0 = const()[name = tensor("input_89_groups_0"), val = tensor(1)]; - tensor input_89 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_89_dilations_0, groups = input_89_groups_0, pad = input_89_pad_0, pad_type = input_89_pad_type_0, strides = input_89_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_87)[name = tensor("input_89")]; - tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_691_begin_0 = const()[name = tensor("op_691_begin_0"), val = tensor([0, -1, 0])]; - tensor var_691_end_0 = const()[name = tensor("op_691_end_0"), val = tensor([1, 16, 256])]; - tensor var_691_end_mask_0 = const()[name = tensor("op_691_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_89)[name = tensor("transpose_55")]; - tensor var_691 = slice_by_index(begin = var_691_begin_0, end = var_691_end_0, end_mask = var_691_end_mask_0, x = conv_out_7)[name = tensor("op_691")]; - tensor var_694_begin_0 = const()[name = tensor("op_694_begin_0"), val = tensor([0, 1, 0])]; - tensor var_694_end_0 = const()[name = tensor("op_694_end_0"), val = tensor([1, 2, 256])]; - tensor var_694_end_mask_0 = const()[name = tensor("op_694_end_mask_0"), val = tensor([true, false, true])]; - tensor var_694 = slice_by_index(begin = var_694_begin_0, end = var_694_end_0, end_mask = var_694_end_mask_0, x = x_13)[name = tensor("op_694")]; - tensor var_697_begin_0 = const()[name = tensor("op_697_begin_0"), val = tensor([0, 1, 0])]; - tensor var_697_end_0 = const()[name = tensor("op_697_end_0"), val = tensor([1, 16, 256])]; - tensor var_697_end_mask_0 = const()[name = tensor("op_697_end_mask_0"), val = tensor([true, true, true])]; - tensor var_697 = slice_by_index(begin = var_697_begin_0, end = var_697_end_0, end_mask = var_697_end_mask_0, x = input_81)[name = tensor("op_697")]; + tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_503)[name = tensor("x_expanded_7")]; + tensor var_517 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_517")]; + tensor var_518 = mul(x = window_9, y = var_517)[name = tensor("op_518")]; + tensor var_519 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_519")]; + tensor filled_window_7 = add(x = var_518, y = var_519)[name = tensor("filled_window_7")]; + tensor var_521 = greater_equal(x = filled_7, y = var_16)[name = tensor("op_521")]; + tensor var_522_dtype_0 = const()[name = tensor("op_522_dtype_0"), val = tensor("fp32")]; + tensor var_523 = const()[name = tensor("op_523"), val = tensor([1, 1, 1])]; + tensor var_522 = cast(dtype = var_522_dtype_0, x = var_521)[name = tensor("cast_152")]; + tensor use_roll_7 = reshape(shape = var_523, x = var_522)[name = tensor("use_roll_7")]; + tensor var_525 = sub(x = var_17, y = use_roll_7)[name = tensor("op_525")]; + tensor var_526 = mul(x = filled_window_7, y = var_525)[name = tensor("op_526")]; + tensor var_527 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_527")]; + tensor window_11 = add(x = var_526, y = var_527)[name = tensor("window_11")]; + tensor var_529 = const()[name = tensor("op_529"), val = tensor(0x1p+0)]; + tensor var_530 = add(x = filled_7, y = var_529)[name = tensor("op_530")]; + tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_530)[name = tensor("clip_7")]; + tensor var_533_begin_0 = const()[name = tensor("op_533_begin_0"), val = tensor([0, 1, 0])]; + tensor var_533_end_0 = const()[name = tensor("op_533_end_0"), val = tensor([1, 2, 256])]; + tensor var_533_end_mask_0 = const()[name = tensor("op_533_end_mask_0"), val = tensor([true, false, true])]; + tensor var_533 = slice_by_index(begin = var_533_begin_0, end = var_533_end_0, end_mask = var_533_end_mask_0, x = x_9)[name = tensor("op_533")]; + tensor var_536_begin_0 = const()[name = tensor("op_536_begin_0"), val = tensor([0, 1, 0])]; + tensor var_536_end_0 = const()[name = tensor("op_536_end_0"), val = tensor([1, 16, 256])]; + tensor var_536_end_mask_0 = const()[name = tensor("op_536_end_mask_0"), val = tensor([true, true, true])]; + tensor var_536 = slice_by_index(begin = var_536_begin_0, end = var_536_end_0, end_mask = var_536_end_mask_0, x = window_11)[name = tensor("op_536")]; tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; - tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_697, var_694))[name = tensor("rolled_9")]; + tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_536, var_533))[name = tensor("rolled_9")]; tensor clip_8 = clip(alpha = var_21, beta = var_20, x = clip_7)[name = tensor("clip_8")]; - tensor var_702 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_702")]; - tensor var_703_dtype_0 = const()[name = tensor("op_703_dtype_0"), val = tensor("fp32")]; - tensor var_704 = const()[name = tensor("op_704"), val = tensor([1, 16, 1])]; - tensor var_703 = cast(dtype = var_703_dtype_0, x = var_702)[name = tensor("cast_151")]; - tensor fill_mask_9 = reshape(shape = var_704, x = var_703)[name = tensor("fill_mask_9")]; + tensor var_541 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_541")]; + tensor var_542_dtype_0 = const()[name = tensor("op_542_dtype_0"), val = tensor("fp32")]; + tensor var_543 = const()[name = tensor("op_543"), val = tensor([1, 16, 1])]; + tensor var_542 = cast(dtype = var_542_dtype_0, x = var_541)[name = tensor("cast_151")]; + tensor fill_mask_9 = reshape(shape = var_543, x = var_542)[name = tensor("fill_mask_9")]; tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_694)[name = tensor("x_expanded_9")]; - tensor var_708 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_708")]; - tensor var_709 = mul(x = input_81, y = var_708)[name = tensor("op_709")]; - tensor var_710 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_710")]; - tensor filled_window_9 = add(x = var_709, y = var_710)[name = tensor("filled_window_9")]; - tensor var_712 = greater_equal(x = clip_7, y = var_16)[name = tensor("op_712")]; - tensor var_713_dtype_0 = const()[name = tensor("op_713_dtype_0"), val = tensor("fp32")]; - tensor var_714 = const()[name = tensor("op_714"), val = tensor([1, 1, 1])]; - tensor var_713 = cast(dtype = var_713_dtype_0, x = var_712)[name = tensor("cast_150")]; - tensor use_roll_9 = reshape(shape = var_714, x = var_713)[name = tensor("use_roll_9")]; - tensor var_716 = sub(x = var_17, y = use_roll_9)[name = tensor("op_716")]; - tensor var_717 = mul(x = filled_window_9, y = var_716)[name = tensor("op_717")]; - tensor var_718 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_718")]; - tensor input_91 = add(x = var_717, y = var_718)[name = tensor("input_91")]; - tensor var_720 = const()[name = tensor("op_720"), val = tensor(0x1p+0)]; - tensor var_721 = add(x = clip_7, y = var_720)[name = tensor("op_721")]; - tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_721)[name = tensor("clip_9")]; - tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; - tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_91)[name = tensor("x_17")]; - tensor input_93_perm_0 = const()[name = tensor("input_93_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_31_pad_type_0 = const()[name = tensor("inputs_31_pad_type_0"), val = tensor("valid")]; - tensor inputs_31_strides_0 = const()[name = tensor("inputs_31_strides_0"), val = tensor([1])]; - tensor inputs_31_pad_0 = const()[name = tensor("inputs_31_pad_0"), val = tensor([0, 0])]; - tensor inputs_31_dilations_0 = const()[name = tensor("inputs_31_dilations_0"), val = tensor([1])]; - tensor inputs_31_groups_0 = const()[name = tensor("inputs_31_groups_0"), val = tensor(1)]; - tensor input_93 = transpose(perm = input_93_perm_0, x = x_17)[name = tensor("transpose_54")]; - tensor inputs_31 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_31_dilations_0, groups = inputs_31_groups_0, pad = inputs_31_pad_0, pad_type = inputs_31_pad_type_0, strides = inputs_31_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_93)[name = tensor("inputs_31")]; - tensor var_742_split_sizes_0 = const()[name = tensor("op_742_split_sizes_0"), val = tensor([256, 256])]; - tensor var_742_axis_0 = const()[name = tensor("op_742_axis_0"), val = tensor(1)]; - tensor var_742_0, tensor var_742_1 = split(axis = var_742_axis_0, split_sizes = var_742_split_sizes_0, x = inputs_31)[name = tensor("op_742")]; - tensor var_744 = sigmoid(x = var_742_1)[name = tensor("op_744")]; - tensor inputs_33 = mul(x = var_742_0, y = var_744)[name = tensor("inputs_33")]; - tensor outputs_aug_9_pad_type_0 = const()[name = tensor("outputs_aug_9_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_9_pad_0 = const()[name = tensor("outputs_aug_9_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_9_groups_0 = const()[name = tensor("outputs_aug_9_groups_0"), val = tensor(256)]; - tensor outputs_aug_9_strides_0 = const()[name = tensor("outputs_aug_9_strides_0"), val = tensor([1])]; - tensor outputs_aug_9_dilations_0 = const()[name = tensor("outputs_aug_9_dilations_0"), val = tensor([1])]; - tensor outputs_aug_9 = conv(dilations = outputs_aug_9_dilations_0, groups = outputs_aug_9_groups_0, pad = outputs_aug_9_pad_0, pad_type = outputs_aug_9_pad_type_0, strides = outputs_aug_9_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_33)[name = tensor("outputs_aug_9")]; - tensor input_95_begin_0 = const()[name = tensor("input_95_begin_0"), val = tensor([0, 0, 0])]; - tensor input_95_end_0 = const()[name = tensor("input_95_end_0"), val = tensor([1, 256, 16])]; - tensor input_95_end_mask_0 = const()[name = tensor("input_95_end_mask_0"), val = tensor([true, true, false])]; - tensor input_95 = slice_by_index(begin = input_95_begin_0, end = input_95_end_0, end_mask = input_95_end_mask_0, x = outputs_aug_9)[name = tensor("input_95")]; - tensor inputs_35 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_95)[name = tensor("inputs_35")]; - tensor input_97 = silu(x = inputs_35)[name = tensor("input_97")]; - tensor input_99_pad_type_0 = const()[name = tensor("input_99_pad_type_0"), val = tensor("valid")]; - tensor input_99_strides_0 = const()[name = tensor("input_99_strides_0"), val = tensor([1])]; - tensor input_99_pad_0 = const()[name = tensor("input_99_pad_0"), val = tensor([0, 0])]; - tensor input_99_dilations_0 = const()[name = tensor("input_99_dilations_0"), val = tensor([1])]; - tensor input_99_groups_0 = const()[name = tensor("input_99_groups_0"), val = tensor(1)]; - tensor input_99 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_99_dilations_0, groups = input_99_groups_0, pad = input_99_pad_0, pad_type = input_99_pad_type_0, strides = input_99_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_97)[name = tensor("input_99")]; - tensor conv_out_9_perm_0 = const()[name = tensor("conv_out_9_perm_0"), val = tensor([0, 2, 1])]; - tensor var_775_begin_0 = const()[name = tensor("op_775_begin_0"), val = tensor([0, -1, 0])]; - tensor var_775_end_0 = const()[name = tensor("op_775_end_0"), val = tensor([1, 16, 256])]; - tensor var_775_end_mask_0 = const()[name = tensor("op_775_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_9 = transpose(perm = conv_out_9_perm_0, x = input_99)[name = tensor("transpose_53")]; - tensor var_775 = slice_by_index(begin = var_775_begin_0, end = var_775_end_0, end_mask = var_775_end_mask_0, x = conv_out_9)[name = tensor("op_775")]; - tensor var_778_begin_0 = const()[name = tensor("op_778_begin_0"), val = tensor([0, 2, 0])]; - tensor var_778_end_0 = const()[name = tensor("op_778_end_0"), val = tensor([1, 1, 256])]; - tensor var_778_end_mask_0 = const()[name = tensor("op_778_end_mask_0"), val = tensor([true, true, true])]; - tensor var_778 = slice_by_index(begin = var_778_begin_0, end = var_778_end_0, end_mask = var_778_end_mask_0, x = x_13)[name = tensor("op_778")]; - tensor var_781_begin_0 = const()[name = tensor("op_781_begin_0"), val = tensor([0, 1, 0])]; - tensor var_781_end_0 = const()[name = tensor("op_781_end_0"), val = tensor([1, 16, 256])]; - tensor var_781_end_mask_0 = const()[name = tensor("op_781_end_mask_0"), val = tensor([true, true, true])]; - tensor var_781 = slice_by_index(begin = var_781_begin_0, end = var_781_end_0, end_mask = var_781_end_mask_0, x = input_91)[name = tensor("op_781")]; + tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_533)[name = tensor("x_expanded_9")]; + tensor var_547 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_547")]; + tensor var_548 = mul(x = window_11, y = var_547)[name = tensor("op_548")]; + tensor var_549 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_549")]; + tensor filled_window_9 = add(x = var_548, y = var_549)[name = tensor("filled_window_9")]; + tensor var_551 = greater_equal(x = clip_7, y = var_16)[name = tensor("op_551")]; + tensor var_552_dtype_0 = const()[name = tensor("op_552_dtype_0"), val = tensor("fp32")]; + tensor var_553 = const()[name = tensor("op_553"), val = tensor([1, 1, 1])]; + tensor var_552 = cast(dtype = var_552_dtype_0, x = var_551)[name = tensor("cast_150")]; + tensor use_roll_9 = reshape(shape = var_553, x = var_552)[name = tensor("use_roll_9")]; + tensor var_555 = sub(x = var_17, y = use_roll_9)[name = tensor("op_555")]; + tensor var_556 = mul(x = filled_window_9, y = var_555)[name = tensor("op_556")]; + tensor var_557 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_557")]; + tensor window_13 = add(x = var_556, y = var_557)[name = tensor("window_13")]; + tensor var_559 = const()[name = tensor("op_559"), val = tensor(0x1p+0)]; + tensor var_560 = add(x = clip_7, y = var_559)[name = tensor("op_560")]; + tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_560)[name = tensor("clip_9")]; + tensor var_563_begin_0 = const()[name = tensor("op_563_begin_0"), val = tensor([0, 2, 0])]; + tensor var_563_end_0 = const()[name = tensor("op_563_end_0"), val = tensor([1, 1, 256])]; + tensor var_563_end_mask_0 = const()[name = tensor("op_563_end_mask_0"), val = tensor([true, true, true])]; + tensor var_563 = slice_by_index(begin = var_563_begin_0, end = var_563_end_0, end_mask = var_563_end_mask_0, x = x_9)[name = tensor("op_563")]; + tensor var_566_begin_0 = const()[name = tensor("op_566_begin_0"), val = tensor([0, 1, 0])]; + tensor var_566_end_0 = const()[name = tensor("op_566_end_0"), val = tensor([1, 16, 256])]; + tensor var_566_end_mask_0 = const()[name = tensor("op_566_end_mask_0"), val = tensor([true, true, true])]; + tensor var_566 = slice_by_index(begin = var_566_begin_0, end = var_566_end_0, end_mask = var_566_end_mask_0, x = window_13)[name = tensor("op_566")]; tensor rolled_11_interleave_0 = const()[name = tensor("rolled_11_interleave_0"), val = tensor(false)]; - tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_781, var_778))[name = tensor("rolled_11")]; + tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_566, var_563))[name = tensor("rolled_11")]; tensor clip_10 = clip(alpha = var_21, beta = var_20, x = clip_9)[name = tensor("clip_10")]; - tensor var_786 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_786")]; - tensor var_787_dtype_0 = const()[name = tensor("op_787_dtype_0"), val = tensor("fp32")]; - tensor var_788 = const()[name = tensor("op_788"), val = tensor([1, 16, 1])]; - tensor var_787 = cast(dtype = var_787_dtype_0, x = var_786)[name = tensor("cast_149")]; - tensor fill_mask_11 = reshape(shape = var_788, x = var_787)[name = tensor("fill_mask_11")]; + tensor var_571 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_571")]; + tensor var_572_dtype_0 = const()[name = tensor("op_572_dtype_0"), val = tensor("fp32")]; + tensor var_573 = const()[name = tensor("op_573"), val = tensor([1, 16, 1])]; + tensor var_572 = cast(dtype = var_572_dtype_0, x = var_571)[name = tensor("cast_149")]; + tensor fill_mask_11 = reshape(shape = var_573, x = var_572)[name = tensor("fill_mask_11")]; tensor x_expanded_11_reps_0 = const()[name = tensor("x_expanded_11_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_778)[name = tensor("x_expanded_11")]; - tensor var_792 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_792")]; - tensor var_793 = mul(x = input_91, y = var_792)[name = tensor("op_793")]; - tensor var_794 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_794")]; - tensor filled_window_11 = add(x = var_793, y = var_794)[name = tensor("filled_window_11")]; - tensor var_796 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_796")]; - tensor var_797_dtype_0 = const()[name = tensor("op_797_dtype_0"), val = tensor("fp32")]; - tensor var_798 = const()[name = tensor("op_798"), val = tensor([1, 1, 1])]; - tensor var_797 = cast(dtype = var_797_dtype_0, x = var_796)[name = tensor("cast_148")]; - tensor use_roll_11 = reshape(shape = var_798, x = var_797)[name = tensor("use_roll_11")]; - tensor var_800 = sub(x = var_17, y = use_roll_11)[name = tensor("op_800")]; - tensor var_801 = mul(x = filled_window_11, y = var_800)[name = tensor("op_801")]; - tensor var_802 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_802")]; - tensor input_101 = add(x = var_801, y = var_802)[name = tensor("input_101")]; - tensor var_804 = const()[name = tensor("op_804"), val = tensor(0x1p+0)]; - tensor var_805 = add(x = clip_9, y = var_804)[name = tensor("op_805")]; - tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_805)[name = tensor("clip_11")]; - tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; - tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_101)[name = tensor("x_19")]; - tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_37_pad_type_0 = const()[name = tensor("inputs_37_pad_type_0"), val = tensor("valid")]; - tensor inputs_37_strides_0 = const()[name = tensor("inputs_37_strides_0"), val = tensor([1])]; - tensor inputs_37_pad_0 = const()[name = tensor("inputs_37_pad_0"), val = tensor([0, 0])]; - tensor inputs_37_dilations_0 = const()[name = tensor("inputs_37_dilations_0"), val = tensor([1])]; - tensor inputs_37_groups_0 = const()[name = tensor("inputs_37_groups_0"), val = tensor(1)]; - tensor input_103 = transpose(perm = input_103_perm_0, x = x_19)[name = tensor("transpose_52")]; - tensor inputs_37 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_37_dilations_0, groups = inputs_37_groups_0, pad = inputs_37_pad_0, pad_type = inputs_37_pad_type_0, strides = inputs_37_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_37")]; - tensor var_826_split_sizes_0 = const()[name = tensor("op_826_split_sizes_0"), val = tensor([256, 256])]; - tensor var_826_axis_0 = const()[name = tensor("op_826_axis_0"), val = tensor(1)]; - tensor var_826_0, tensor var_826_1 = split(axis = var_826_axis_0, split_sizes = var_826_split_sizes_0, x = inputs_37)[name = tensor("op_826")]; - tensor var_828 = sigmoid(x = var_826_1)[name = tensor("op_828")]; - tensor inputs_39 = mul(x = var_826_0, y = var_828)[name = tensor("inputs_39")]; - tensor outputs_aug_11_pad_type_0 = const()[name = tensor("outputs_aug_11_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_11_pad_0 = const()[name = tensor("outputs_aug_11_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_11_groups_0 = const()[name = tensor("outputs_aug_11_groups_0"), val = tensor(256)]; - tensor outputs_aug_11_strides_0 = const()[name = tensor("outputs_aug_11_strides_0"), val = tensor([1])]; - tensor outputs_aug_11_dilations_0 = const()[name = tensor("outputs_aug_11_dilations_0"), val = tensor([1])]; - tensor outputs_aug_11 = conv(dilations = outputs_aug_11_dilations_0, groups = outputs_aug_11_groups_0, pad = outputs_aug_11_pad_0, pad_type = outputs_aug_11_pad_type_0, strides = outputs_aug_11_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_39)[name = tensor("outputs_aug_11")]; - tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; - tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([1, 256, 16])]; - tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; - tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_11)[name = tensor("input_105")]; - tensor inputs_41 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_105)[name = tensor("inputs_41")]; - tensor input_107 = silu(x = inputs_41)[name = tensor("input_107")]; - tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; - tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; - tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; - tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; - tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; - tensor input_109 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; - tensor conv_out_11_perm_0 = const()[name = tensor("conv_out_11_perm_0"), val = tensor([0, 2, 1])]; - tensor var_859_begin_0 = const()[name = tensor("op_859_begin_0"), val = tensor([0, -1, 0])]; - tensor var_859_end_0 = const()[name = tensor("op_859_end_0"), val = tensor([1, 16, 256])]; - tensor var_859_end_mask_0 = const()[name = tensor("op_859_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_11 = transpose(perm = conv_out_11_perm_0, x = input_109)[name = tensor("transpose_51")]; - tensor var_859 = slice_by_index(begin = var_859_begin_0, end = var_859_end_0, end_mask = var_859_end_mask_0, x = conv_out_11)[name = tensor("op_859")]; - tensor var_862_interleave_0 = const()[name = tensor("op_862_interleave_0"), val = tensor(false)]; - tensor var_862 = concat(axis = var_33, interleave = var_862_interleave_0, values = (var_691, var_775, var_859))[name = tensor("op_862")]; - tensor input_111 = add(x = x_13, y = var_862)[name = tensor("input_111")]; - tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; - tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; - tensor inputs_43 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_17")]; - tensor input_115 = silu(x = inputs_43)[name = tensor("input_115")]; - tensor input_119 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_18")]; - tensor var_885 = const()[name = tensor("op_885"), val = tensor(0x1p-1)]; - tensor var_886 = mul(x = input_119, y = var_885)[name = tensor("op_886")]; - tensor input_121 = add(x = var_886, y = input_111)[name = tensor("input_121")]; - tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; - tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_121)[name = tensor("input_123")]; - tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; - tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; - tensor inputs_45 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_19")]; - tensor input_127 = silu(x = inputs_45)[name = tensor("input_127")]; - tensor input_131 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_20")]; - tensor var_915 = const()[name = tensor("op_915"), val = tensor(0x1p-1)]; - tensor var_916 = mul(x = input_131, y = var_915)[name = tensor("op_916")]; - tensor input_133 = add(x = var_916, y = input_123)[name = tensor("input_133")]; - tensor x_21_axes_0 = const()[name = tensor("x_21_axes_0"), val = tensor([-1])]; - tensor x_21 = layer_norm(axes = x_21_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_133)[name = tensor("x_21")]; + tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_563)[name = tensor("x_expanded_11")]; + tensor var_577 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_577")]; + tensor var_578 = mul(x = window_13, y = var_577)[name = tensor("op_578")]; + tensor var_579 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_579")]; + tensor filled_window_11 = add(x = var_578, y = var_579)[name = tensor("filled_window_11")]; + tensor var_581 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_581")]; + tensor var_582_dtype_0 = const()[name = tensor("op_582_dtype_0"), val = tensor("fp32")]; + tensor var_583 = const()[name = tensor("op_583"), val = tensor([1, 1, 1])]; + tensor var_582 = cast(dtype = var_582_dtype_0, x = var_581)[name = tensor("cast_148")]; + tensor use_roll_11 = reshape(shape = var_583, x = var_582)[name = tensor("use_roll_11")]; + tensor var_585 = sub(x = var_17, y = use_roll_11)[name = tensor("op_585")]; + tensor var_586 = mul(x = filled_window_11, y = var_585)[name = tensor("op_586")]; + tensor var_587 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_587")]; + tensor window_15 = add(x = var_586, y = var_587)[name = tensor("window_15")]; + tensor var_589 = const()[name = tensor("op_589"), val = tensor(0x1p+0)]; + tensor var_590 = add(x = clip_9, y = var_589)[name = tensor("op_590")]; + tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_590)[name = tensor("clip_11")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_30, interleave = input_61_interleave_0, values = (window_11, window_13, window_15))[name = tensor("input_61")]; + tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; + tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; + tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_13_pad_type_0 = const()[name = tensor("inputs_13_pad_type_0"), val = tensor("valid")]; + tensor inputs_13_strides_0 = const()[name = tensor("inputs_13_strides_0"), val = tensor([1])]; + tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; + tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; + tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; + tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; + tensor var_613_split_sizes_0 = const()[name = tensor("op_613_split_sizes_0"), val = tensor([256, 256])]; + tensor var_613_axis_0 = const()[name = tensor("op_613_axis_0"), val = tensor(1)]; + tensor var_613_0, tensor var_613_1 = split(axis = var_613_axis_0, split_sizes = var_613_split_sizes_0, x = inputs_13)[name = tensor("op_613")]; + tensor var_615 = sigmoid(x = var_613_1)[name = tensor("op_615")]; + tensor inputs_15 = mul(x = var_613_0, y = var_615)[name = tensor("inputs_15")]; + tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; + tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; + tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; + tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_15)[name = tensor("outputs_aug_3")]; + tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; + tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([3, 256, 16])]; + tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; + tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_3)[name = tensor("input_65")]; + tensor inputs_17 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_65)[name = tensor("inputs_17")]; + tensor input_67 = silu(x = inputs_17)[name = tensor("input_67")]; + tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; + tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; + tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; + tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; + tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; + tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; + tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; + tensor var_646_begin_0 = const()[name = tensor("op_646_begin_0"), val = tensor([0, -1, 0])]; + tensor var_646_end_0 = const()[name = tensor("op_646_end_0"), val = tensor([3, 16, 256])]; + tensor var_646_end_mask_0 = const()[name = tensor("op_646_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_646 = slice_by_index(begin = var_646_begin_0, end = var_646_end_0, end_mask = var_646_end_mask_0, x = conv_out_3)[name = tensor("op_646")]; + tensor var_648_perm_0 = const()[name = tensor("op_648_perm_0"), val = tensor([1, 0, 2])]; + tensor var_648 = transpose(perm = var_648_perm_0, x = var_646)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_648)[name = tensor("input_71")]; + tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; + tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; + tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; + tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; + tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; + tensor var_671 = const()[name = tensor("op_671"), val = tensor(0x1p-1)]; + tensor var_672 = mul(x = input_79, y = var_671)[name = tensor("op_672")]; + tensor input_81 = add(x = var_672, y = input_71)[name = tensor("input_81")]; + tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; + tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; + tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; + tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; + tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; + tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; + tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; + tensor var_701 = const()[name = tensor("op_701"), val = tensor(0x1p-1)]; + tensor var_702 = mul(x = input_91, y = var_701)[name = tensor("op_702")]; + tensor input_93 = add(x = var_702, y = input_83)[name = tensor("input_93")]; + tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; + tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; tensor prev_kv_5_end_0 = const()[name = tensor("prev_kv_5_end_0"), val = tensor([3, 1, 4, 64, 64])]; tensor prev_kv_5_end_mask_0 = const()[name = tensor("prev_kv_5_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -883,316 +732,240 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_930 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_21)[name = tensor("linear_21")]; - tensor var_931 = const()[name = tensor("op_931"), val = tensor([1, 3, 4, 64])]; - tensor var_932 = reshape(shape = var_931, x = var_930)[name = tensor("op_932")]; + tensor var_716 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_717 = const()[name = tensor("op_717"), val = tensor([1, 3, 4, 64])]; + tensor var_718 = reshape(shape = var_717, x = var_716)[name = tensor("op_718")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_936 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_21)[name = tensor("linear_22")]; - tensor var_937 = const()[name = tensor("op_937"), val = tensor(0x1p-3)]; - tensor var_938 = mul(x = var_936, y = var_937)[name = tensor("op_938")]; - tensor var_939 = const()[name = tensor("op_939"), val = tensor([1, 3, 4, 64])]; - tensor var_940 = reshape(shape = var_939, x = var_938)[name = tensor("op_940")]; + tensor var_722 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_723 = const()[name = tensor("op_723"), val = tensor(0x1p-3)]; + tensor var_724 = mul(x = var_722, y = var_723)[name = tensor("op_724")]; + tensor var_725 = const()[name = tensor("op_725"), val = tensor([1, 3, 4, 64])]; + tensor var_726 = reshape(shape = var_725, x = var_724)[name = tensor("op_726")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_944 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_21)[name = tensor("linear_23")]; - tensor var_945 = const()[name = tensor("op_945"), val = tensor([1, 3, 4, 64])]; - tensor var_946 = reshape(shape = var_945, x = var_944)[name = tensor("op_946")]; + tensor var_730 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_731 = const()[name = tensor("op_731"), val = tensor([1, 3, 4, 64])]; + tensor var_732 = reshape(shape = var_731, x = var_730)[name = tensor("op_732")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_137 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_21)[name = tensor("linear_24")]; + tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; tensor s_t_5 = add(x = prev_scale_5, y = encoder__t_index)[name = tensor("s_t_5")]; tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_940)[name = tensor("transpose_49")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_932)[name = tensor("transpose_50")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_726)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_718)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_956 = const()[name = tensor("op_956"), val = tensor([3, 1])]; - tensor var_957 = reshape(shape = var_956, x = sqrt_s_t_5)[name = tensor("op_957")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_957)[name = tensor("M_5")]; - tensor var_959 = mul(x = qk_5, y = M_5)[name = tensor("op_959")]; + tensor var_742 = const()[name = tensor("op_742"), val = tensor([3, 1])]; + tensor var_743 = reshape(shape = var_742, x = sqrt_s_t_5)[name = tensor("op_743")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_743)[name = tensor("M_5")]; + tensor var_745 = mul(x = qk_5, y = M_5)[name = tensor("op_745")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_946)[name = tensor("transpose_48")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_959, y = v_5)[name = tensor("inner_5")]; - tensor var_961_transpose_x_0 = const()[name = tensor("op_961_transpose_x_0"), val = tensor(false)]; - tensor var_961_transpose_y_0 = const()[name = tensor("op_961_transpose_y_0"), val = tensor(false)]; - tensor var_961 = matmul(transpose_x = var_961_transpose_x_0, transpose_y = var_961_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_961")]; - tensor var_962 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_962")]; - tensor var_963 = const()[name = tensor("op_963"), val = tensor([1, 1, 3, 1])]; - tensor var_964 = reshape(shape = var_963, x = var_962)[name = tensor("op_964")]; - tensor cross_5 = mul(x = var_961, y = var_964)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_732)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_745, y = v_5)[name = tensor("inner_5")]; + tensor var_747_transpose_x_0 = const()[name = tensor("op_747_transpose_x_0"), val = tensor(false)]; + tensor var_747_transpose_y_0 = const()[name = tensor("op_747_transpose_y_0"), val = tensor(false)]; + tensor var_747 = matmul(transpose_x = var_747_transpose_x_0, transpose_y = var_747_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_747")]; + tensor var_748 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_748")]; + tensor var_749 = const()[name = tensor("op_749"), val = tensor([1, 1, 3, 1])]; + tensor var_750 = reshape(shape = var_749, x = var_748)[name = tensor("op_750")]; + tensor cross_5 = mul(x = var_747, y = var_750)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_967 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_967")]; - tensor var_969_transpose_x_1 = const()[name = tensor("op_969_transpose_x_1"), val = tensor(true)]; - tensor var_969_transpose_y_1 = const()[name = tensor("op_969_transpose_y_1"), val = tensor(false)]; - tensor var_969 = matmul(transpose_x = var_969_transpose_x_1, transpose_y = var_969_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_969")]; - tensor new_kv_unnorm_5 = add(x = var_967, y = var_969)[name = tensor("new_kv_unnorm_5")]; - tensor var_971 = const()[name = tensor("op_971"), val = tensor(0x1.8p+1)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_971)[name = tensor("new_scale_5")]; - tensor var_973 = sqrt(x = new_scale_5)[name = tensor("op_973")]; - tensor var_974 = real_div(x = new_kv_unnorm_5, y = var_973)[name = tensor("op_974")]; - tensor var_975_perm_0 = const()[name = tensor("op_975_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_753 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_753")]; + tensor var_755_transpose_x_1 = const()[name = tensor("op_755_transpose_x_1"), val = tensor(true)]; + tensor var_755_transpose_y_1 = const()[name = tensor("op_755_transpose_y_1"), val = tensor(false)]; + tensor var_755 = matmul(transpose_x = var_755_transpose_x_1, transpose_y = var_755_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_755")]; + tensor new_kv_unnorm_5 = add(x = var_753, y = var_755)[name = tensor("new_kv_unnorm_5")]; + tensor var_757 = const()[name = tensor("op_757"), val = tensor(0x1.8p+1)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_757)[name = tensor("new_scale_5")]; + tensor var_759 = sqrt(x = new_scale_5)[name = tensor("op_759")]; + tensor var_760 = real_div(x = new_kv_unnorm_5, y = var_759)[name = tensor("op_760")]; + tensor var_761_perm_0 = const()[name = tensor("op_761_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_975 = transpose(perm = var_975_perm_0, x = out_13)[name = tensor("transpose_47")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_975)[name = tensor("out_15")]; - tensor var_979 = const()[name = tensor("op_979"), val = tensor([1, 3, 256])]; - tensor out_17 = reshape(shape = var_979, x = out_15)[name = tensor("out_17")]; - tensor var_981 = silu(x = input_137)[name = tensor("op_981")]; - tensor input_139 = mul(x = var_981, y = out_17)[name = tensor("input_139")]; - tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_139)[name = tensor("linear_25")]; - tensor x_23 = add(x = input_133, y = ret_out_5)[name = tensor("x_23")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor var_761 = transpose(perm = var_761_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_761)[name = tensor("out_15")]; + tensor var_765 = const()[name = tensor("op_765"), val = tensor([1, 3, 256])]; + tensor out_17 = reshape(shape = var_765, x = out_15)[name = tensor("out_17")]; + tensor var_767 = silu(x = input_97)[name = tensor("op_767")]; + tensor input_99 = mul(x = var_767, y = out_17)[name = tensor("input_99")]; + tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; + tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; + tensor window_17_begin_0 = const()[name = tensor("window_17_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_17_end_0 = const()[name = tensor("window_17_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_17_end_mask_0 = const()[name = tensor("window_17_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_17_squeeze_mask_0 = const()[name = tensor("window_17_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_17 = slice_by_index(begin = window_17_begin_0, end = window_17_end_0, end_mask = window_17_end_mask_0, squeeze_mask = window_17_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_17")]; tensor filled_13_begin_0 = const()[name = tensor("filled_13_begin_0"), val = tensor([2, 0])]; tensor filled_13_end_0 = const()[name = tensor("filled_13_end_0"), val = tensor([3, 1])]; tensor filled_13_end_mask_0 = const()[name = tensor("filled_13_end_mask_0"), val = tensor([false, true])]; tensor filled_13_squeeze_mask_0 = const()[name = tensor("filled_13_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_13 = slice_by_index(begin = filled_13_begin_0, end = filled_13_end_0, end_mask = filled_13_end_mask_0, squeeze_mask = filled_13_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_13")]; - tensor var_990_begin_0 = const()[name = tensor("op_990_begin_0"), val = tensor([0, 0, 0])]; - tensor var_990_end_0 = const()[name = tensor("op_990_end_0"), val = tensor([1, 1, 256])]; - tensor var_990_end_mask_0 = const()[name = tensor("op_990_end_mask_0"), val = tensor([true, false, true])]; - tensor var_990 = slice_by_index(begin = var_990_begin_0, end = var_990_end_0, end_mask = var_990_end_mask_0, x = x_23)[name = tensor("op_990")]; - tensor var_993_begin_0 = const()[name = tensor("op_993_begin_0"), val = tensor([0, 1, 0])]; - tensor var_993_end_0 = const()[name = tensor("op_993_end_0"), val = tensor([1, 16, 256])]; - tensor var_993_end_mask_0 = const()[name = tensor("op_993_end_mask_0"), val = tensor([true, true, true])]; - tensor var_993 = slice_by_index(begin = var_993_begin_0, end = var_993_end_0, end_mask = var_993_end_mask_0, x = window_5)[name = tensor("op_993")]; + tensor var_776_begin_0 = const()[name = tensor("op_776_begin_0"), val = tensor([0, 0, 0])]; + tensor var_776_end_0 = const()[name = tensor("op_776_end_0"), val = tensor([1, 1, 256])]; + tensor var_776_end_mask_0 = const()[name = tensor("op_776_end_mask_0"), val = tensor([true, false, true])]; + tensor var_776 = slice_by_index(begin = var_776_begin_0, end = var_776_end_0, end_mask = var_776_end_mask_0, x = x_15)[name = tensor("op_776")]; + tensor var_779_begin_0 = const()[name = tensor("op_779_begin_0"), val = tensor([0, 1, 0])]; + tensor var_779_end_0 = const()[name = tensor("op_779_end_0"), val = tensor([1, 16, 256])]; + tensor var_779_end_mask_0 = const()[name = tensor("op_779_end_mask_0"), val = tensor([true, true, true])]; + tensor var_779 = slice_by_index(begin = var_779_begin_0, end = var_779_end_0, end_mask = var_779_end_mask_0, x = window_17)[name = tensor("op_779")]; tensor rolled_13_interleave_0 = const()[name = tensor("rolled_13_interleave_0"), val = tensor(false)]; - tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_993, var_990))[name = tensor("rolled_13")]; + tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_779, var_776))[name = tensor("rolled_13")]; tensor clip_12 = clip(alpha = var_21, beta = var_20, x = filled_13)[name = tensor("clip_12")]; - tensor var_998 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_998")]; - tensor var_999_dtype_0 = const()[name = tensor("op_999_dtype_0"), val = tensor("fp32")]; - tensor var_1000 = const()[name = tensor("op_1000"), val = tensor([1, 16, 1])]; - tensor var_999 = cast(dtype = var_999_dtype_0, x = var_998)[name = tensor("cast_147")]; - tensor fill_mask_13 = reshape(shape = var_1000, x = var_999)[name = tensor("fill_mask_13")]; + tensor var_784 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_784")]; + tensor var_785_dtype_0 = const()[name = tensor("op_785_dtype_0"), val = tensor("fp32")]; + tensor var_786 = const()[name = tensor("op_786"), val = tensor([1, 16, 1])]; + tensor var_785 = cast(dtype = var_785_dtype_0, x = var_784)[name = tensor("cast_147")]; + tensor fill_mask_13 = reshape(shape = var_786, x = var_785)[name = tensor("fill_mask_13")]; tensor x_expanded_13_reps_0 = const()[name = tensor("x_expanded_13_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_990)[name = tensor("x_expanded_13")]; - tensor var_1004 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_1004")]; - tensor var_1005 = mul(x = window_5, y = var_1004)[name = tensor("op_1005")]; - tensor var_1006 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_1006")]; - tensor filled_window_13 = add(x = var_1005, y = var_1006)[name = tensor("filled_window_13")]; - tensor var_1008 = greater_equal(x = filled_13, y = var_16)[name = tensor("op_1008")]; - tensor var_1009_dtype_0 = const()[name = tensor("op_1009_dtype_0"), val = tensor("fp32")]; - tensor var_1010 = const()[name = tensor("op_1010"), val = tensor([1, 1, 1])]; - tensor var_1009 = cast(dtype = var_1009_dtype_0, x = var_1008)[name = tensor("cast_146")]; - tensor use_roll_13 = reshape(shape = var_1010, x = var_1009)[name = tensor("use_roll_13")]; - tensor var_1012 = sub(x = var_17, y = use_roll_13)[name = tensor("op_1012")]; - tensor var_1013 = mul(x = filled_window_13, y = var_1012)[name = tensor("op_1013")]; - tensor var_1014 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_1014")]; - tensor input_141 = add(x = var_1013, y = var_1014)[name = tensor("input_141")]; - tensor var_1016 = const()[name = tensor("op_1016"), val = tensor(0x1p+0)]; - tensor var_1017 = add(x = filled_13, y = var_1016)[name = tensor("op_1017")]; - tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_1017)[name = tensor("clip_13")]; - tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; - tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_141)[name = tensor("x_25")]; - tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_47_pad_type_0 = const()[name = tensor("inputs_47_pad_type_0"), val = tensor("valid")]; - tensor inputs_47_strides_0 = const()[name = tensor("inputs_47_strides_0"), val = tensor([1])]; - tensor inputs_47_pad_0 = const()[name = tensor("inputs_47_pad_0"), val = tensor([0, 0])]; - tensor inputs_47_dilations_0 = const()[name = tensor("inputs_47_dilations_0"), val = tensor([1])]; - tensor inputs_47_groups_0 = const()[name = tensor("inputs_47_groups_0"), val = tensor(1)]; - tensor input_143 = transpose(perm = input_143_perm_0, x = x_25)[name = tensor("transpose_46")]; - tensor inputs_47 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_47_dilations_0, groups = inputs_47_groups_0, pad = inputs_47_pad_0, pad_type = inputs_47_pad_type_0, strides = inputs_47_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_47")]; - tensor var_1038_split_sizes_0 = const()[name = tensor("op_1038_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1038_axis_0 = const()[name = tensor("op_1038_axis_0"), val = tensor(1)]; - tensor var_1038_0, tensor var_1038_1 = split(axis = var_1038_axis_0, split_sizes = var_1038_split_sizes_0, x = inputs_47)[name = tensor("op_1038")]; - tensor var_1040 = sigmoid(x = var_1038_1)[name = tensor("op_1040")]; - tensor inputs_49 = mul(x = var_1038_0, y = var_1040)[name = tensor("inputs_49")]; - tensor outputs_aug_13_pad_type_0 = const()[name = tensor("outputs_aug_13_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_13_pad_0 = const()[name = tensor("outputs_aug_13_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_13_groups_0 = const()[name = tensor("outputs_aug_13_groups_0"), val = tensor(256)]; - tensor outputs_aug_13_strides_0 = const()[name = tensor("outputs_aug_13_strides_0"), val = tensor([1])]; - tensor outputs_aug_13_dilations_0 = const()[name = tensor("outputs_aug_13_dilations_0"), val = tensor([1])]; - tensor outputs_aug_13 = conv(dilations = outputs_aug_13_dilations_0, groups = outputs_aug_13_groups_0, pad = outputs_aug_13_pad_0, pad_type = outputs_aug_13_pad_type_0, strides = outputs_aug_13_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_49)[name = tensor("outputs_aug_13")]; - tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; - tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([1, 256, 16])]; - tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; - tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug_13)[name = tensor("input_145")]; - tensor inputs_51 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_145)[name = tensor("inputs_51")]; - tensor input_147 = silu(x = inputs_51)[name = tensor("input_147")]; - tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; - tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; - tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; - tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; - tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; - tensor input_149 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; - tensor conv_out_13_perm_0 = const()[name = tensor("conv_out_13_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1071_begin_0 = const()[name = tensor("op_1071_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1071_end_0 = const()[name = tensor("op_1071_end_0"), val = tensor([1, 16, 256])]; - tensor var_1071_end_mask_0 = const()[name = tensor("op_1071_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_13 = transpose(perm = conv_out_13_perm_0, x = input_149)[name = tensor("transpose_45")]; - tensor var_1071 = slice_by_index(begin = var_1071_begin_0, end = var_1071_end_0, end_mask = var_1071_end_mask_0, x = conv_out_13)[name = tensor("op_1071")]; - tensor var_1074_begin_0 = const()[name = tensor("op_1074_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1074_end_0 = const()[name = tensor("op_1074_end_0"), val = tensor([1, 2, 256])]; - tensor var_1074_end_mask_0 = const()[name = tensor("op_1074_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1074 = slice_by_index(begin = var_1074_begin_0, end = var_1074_end_0, end_mask = var_1074_end_mask_0, x = x_23)[name = tensor("op_1074")]; - tensor var_1077_begin_0 = const()[name = tensor("op_1077_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1077_end_0 = const()[name = tensor("op_1077_end_0"), val = tensor([1, 16, 256])]; - tensor var_1077_end_mask_0 = const()[name = tensor("op_1077_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1077 = slice_by_index(begin = var_1077_begin_0, end = var_1077_end_0, end_mask = var_1077_end_mask_0, x = input_141)[name = tensor("op_1077")]; + tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_776)[name = tensor("x_expanded_13")]; + tensor var_790 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_790")]; + tensor var_791 = mul(x = window_17, y = var_790)[name = tensor("op_791")]; + tensor var_792 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_792")]; + tensor filled_window_13 = add(x = var_791, y = var_792)[name = tensor("filled_window_13")]; + tensor var_794 = greater_equal(x = filled_13, y = var_16)[name = tensor("op_794")]; + tensor var_795_dtype_0 = const()[name = tensor("op_795_dtype_0"), val = tensor("fp32")]; + tensor var_796 = const()[name = tensor("op_796"), val = tensor([1, 1, 1])]; + tensor var_795 = cast(dtype = var_795_dtype_0, x = var_794)[name = tensor("cast_146")]; + tensor use_roll_13 = reshape(shape = var_796, x = var_795)[name = tensor("use_roll_13")]; + tensor var_798 = sub(x = var_17, y = use_roll_13)[name = tensor("op_798")]; + tensor var_799 = mul(x = filled_window_13, y = var_798)[name = tensor("op_799")]; + tensor var_800 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_800")]; + tensor window_19 = add(x = var_799, y = var_800)[name = tensor("window_19")]; + tensor var_802 = const()[name = tensor("op_802"), val = tensor(0x1p+0)]; + tensor var_803 = add(x = filled_13, y = var_802)[name = tensor("op_803")]; + tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_803)[name = tensor("clip_13")]; + tensor var_806_begin_0 = const()[name = tensor("op_806_begin_0"), val = tensor([0, 1, 0])]; + tensor var_806_end_0 = const()[name = tensor("op_806_end_0"), val = tensor([1, 2, 256])]; + tensor var_806_end_mask_0 = const()[name = tensor("op_806_end_mask_0"), val = tensor([true, false, true])]; + tensor var_806 = slice_by_index(begin = var_806_begin_0, end = var_806_end_0, end_mask = var_806_end_mask_0, x = x_15)[name = tensor("op_806")]; + tensor var_809_begin_0 = const()[name = tensor("op_809_begin_0"), val = tensor([0, 1, 0])]; + tensor var_809_end_0 = const()[name = tensor("op_809_end_0"), val = tensor([1, 16, 256])]; + tensor var_809_end_mask_0 = const()[name = tensor("op_809_end_mask_0"), val = tensor([true, true, true])]; + tensor var_809 = slice_by_index(begin = var_809_begin_0, end = var_809_end_0, end_mask = var_809_end_mask_0, x = window_19)[name = tensor("op_809")]; tensor rolled_15_interleave_0 = const()[name = tensor("rolled_15_interleave_0"), val = tensor(false)]; - tensor rolled_15 = concat(axis = var_33, interleave = rolled_15_interleave_0, values = (var_1077, var_1074))[name = tensor("rolled_15")]; + tensor rolled_15 = concat(axis = var_33, interleave = rolled_15_interleave_0, values = (var_809, var_806))[name = tensor("rolled_15")]; tensor clip_14 = clip(alpha = var_21, beta = var_20, x = clip_13)[name = tensor("clip_14")]; - tensor var_1082 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_1082")]; - tensor var_1083_dtype_0 = const()[name = tensor("op_1083_dtype_0"), val = tensor("fp32")]; - tensor var_1084 = const()[name = tensor("op_1084"), val = tensor([1, 16, 1])]; - tensor var_1083 = cast(dtype = var_1083_dtype_0, x = var_1082)[name = tensor("cast_145")]; - tensor fill_mask_15 = reshape(shape = var_1084, x = var_1083)[name = tensor("fill_mask_15")]; + tensor var_814 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_814")]; + tensor var_815_dtype_0 = const()[name = tensor("op_815_dtype_0"), val = tensor("fp32")]; + tensor var_816 = const()[name = tensor("op_816"), val = tensor([1, 16, 1])]; + tensor var_815 = cast(dtype = var_815_dtype_0, x = var_814)[name = tensor("cast_145")]; + tensor fill_mask_15 = reshape(shape = var_816, x = var_815)[name = tensor("fill_mask_15")]; tensor x_expanded_15_reps_0 = const()[name = tensor("x_expanded_15_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_1074)[name = tensor("x_expanded_15")]; - tensor var_1088 = sub(x = var_17, y = fill_mask_15)[name = tensor("op_1088")]; - tensor var_1089 = mul(x = input_141, y = var_1088)[name = tensor("op_1089")]; - tensor var_1090 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_1090")]; - tensor filled_window_15 = add(x = var_1089, y = var_1090)[name = tensor("filled_window_15")]; - tensor var_1092 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_1092")]; - tensor var_1093_dtype_0 = const()[name = tensor("op_1093_dtype_0"), val = tensor("fp32")]; - tensor var_1094 = const()[name = tensor("op_1094"), val = tensor([1, 1, 1])]; - tensor var_1093 = cast(dtype = var_1093_dtype_0, x = var_1092)[name = tensor("cast_144")]; - tensor use_roll_15 = reshape(shape = var_1094, x = var_1093)[name = tensor("use_roll_15")]; - tensor var_1096 = sub(x = var_17, y = use_roll_15)[name = tensor("op_1096")]; - tensor var_1097 = mul(x = filled_window_15, y = var_1096)[name = tensor("op_1097")]; - tensor var_1098 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_1098")]; - tensor input_151 = add(x = var_1097, y = var_1098)[name = tensor("input_151")]; - tensor var_1100 = const()[name = tensor("op_1100"), val = tensor(0x1p+0)]; - tensor var_1101 = add(x = clip_13, y = var_1100)[name = tensor("op_1101")]; - tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_1101)[name = tensor("clip_15")]; - tensor x_27_axes_0 = const()[name = tensor("x_27_axes_0"), val = tensor([-1])]; - tensor x_27 = layer_norm(axes = x_27_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_151)[name = tensor("x_27")]; - tensor input_153_perm_0 = const()[name = tensor("input_153_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_53_pad_type_0 = const()[name = tensor("inputs_53_pad_type_0"), val = tensor("valid")]; - tensor inputs_53_strides_0 = const()[name = tensor("inputs_53_strides_0"), val = tensor([1])]; - tensor inputs_53_pad_0 = const()[name = tensor("inputs_53_pad_0"), val = tensor([0, 0])]; - tensor inputs_53_dilations_0 = const()[name = tensor("inputs_53_dilations_0"), val = tensor([1])]; - tensor inputs_53_groups_0 = const()[name = tensor("inputs_53_groups_0"), val = tensor(1)]; - tensor input_153 = transpose(perm = input_153_perm_0, x = x_27)[name = tensor("transpose_44")]; - tensor inputs_53 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_53_dilations_0, groups = inputs_53_groups_0, pad = inputs_53_pad_0, pad_type = inputs_53_pad_type_0, strides = inputs_53_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_153)[name = tensor("inputs_53")]; - tensor var_1122_split_sizes_0 = const()[name = tensor("op_1122_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1122_axis_0 = const()[name = tensor("op_1122_axis_0"), val = tensor(1)]; - tensor var_1122_0, tensor var_1122_1 = split(axis = var_1122_axis_0, split_sizes = var_1122_split_sizes_0, x = inputs_53)[name = tensor("op_1122")]; - tensor var_1124 = sigmoid(x = var_1122_1)[name = tensor("op_1124")]; - tensor inputs_55 = mul(x = var_1122_0, y = var_1124)[name = tensor("inputs_55")]; - tensor outputs_aug_15_pad_type_0 = const()[name = tensor("outputs_aug_15_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_15_pad_0 = const()[name = tensor("outputs_aug_15_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_15_groups_0 = const()[name = tensor("outputs_aug_15_groups_0"), val = tensor(256)]; - tensor outputs_aug_15_strides_0 = const()[name = tensor("outputs_aug_15_strides_0"), val = tensor([1])]; - tensor outputs_aug_15_dilations_0 = const()[name = tensor("outputs_aug_15_dilations_0"), val = tensor([1])]; - tensor outputs_aug_15 = conv(dilations = outputs_aug_15_dilations_0, groups = outputs_aug_15_groups_0, pad = outputs_aug_15_pad_0, pad_type = outputs_aug_15_pad_type_0, strides = outputs_aug_15_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_55)[name = tensor("outputs_aug_15")]; - tensor input_155_begin_0 = const()[name = tensor("input_155_begin_0"), val = tensor([0, 0, 0])]; - tensor input_155_end_0 = const()[name = tensor("input_155_end_0"), val = tensor([1, 256, 16])]; - tensor input_155_end_mask_0 = const()[name = tensor("input_155_end_mask_0"), val = tensor([true, true, false])]; - tensor input_155 = slice_by_index(begin = input_155_begin_0, end = input_155_end_0, end_mask = input_155_end_mask_0, x = outputs_aug_15)[name = tensor("input_155")]; - tensor inputs_57 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_155)[name = tensor("inputs_57")]; - tensor input_157 = silu(x = inputs_57)[name = tensor("input_157")]; - tensor input_159_pad_type_0 = const()[name = tensor("input_159_pad_type_0"), val = tensor("valid")]; - tensor input_159_strides_0 = const()[name = tensor("input_159_strides_0"), val = tensor([1])]; - tensor input_159_pad_0 = const()[name = tensor("input_159_pad_0"), val = tensor([0, 0])]; - tensor input_159_dilations_0 = const()[name = tensor("input_159_dilations_0"), val = tensor([1])]; - tensor input_159_groups_0 = const()[name = tensor("input_159_groups_0"), val = tensor(1)]; - tensor input_159 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_159_dilations_0, groups = input_159_groups_0, pad = input_159_pad_0, pad_type = input_159_pad_type_0, strides = input_159_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_157)[name = tensor("input_159")]; - tensor conv_out_15_perm_0 = const()[name = tensor("conv_out_15_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1155_begin_0 = const()[name = tensor("op_1155_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1155_end_0 = const()[name = tensor("op_1155_end_0"), val = tensor([1, 16, 256])]; - tensor var_1155_end_mask_0 = const()[name = tensor("op_1155_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_15 = transpose(perm = conv_out_15_perm_0, x = input_159)[name = tensor("transpose_43")]; - tensor var_1155 = slice_by_index(begin = var_1155_begin_0, end = var_1155_end_0, end_mask = var_1155_end_mask_0, x = conv_out_15)[name = tensor("op_1155")]; - tensor var_1158_begin_0 = const()[name = tensor("op_1158_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1158_end_0 = const()[name = tensor("op_1158_end_0"), val = tensor([1, 1, 256])]; - tensor var_1158_end_mask_0 = const()[name = tensor("op_1158_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1158 = slice_by_index(begin = var_1158_begin_0, end = var_1158_end_0, end_mask = var_1158_end_mask_0, x = x_23)[name = tensor("op_1158")]; - tensor var_1161_begin_0 = const()[name = tensor("op_1161_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1161_end_0 = const()[name = tensor("op_1161_end_0"), val = tensor([1, 16, 256])]; - tensor var_1161_end_mask_0 = const()[name = tensor("op_1161_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1161 = slice_by_index(begin = var_1161_begin_0, end = var_1161_end_0, end_mask = var_1161_end_mask_0, x = input_151)[name = tensor("op_1161")]; + tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_806)[name = tensor("x_expanded_15")]; + tensor var_820 = sub(x = var_17, y = fill_mask_15)[name = tensor("op_820")]; + tensor var_821 = mul(x = window_19, y = var_820)[name = tensor("op_821")]; + tensor var_822 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_822")]; + tensor filled_window_15 = add(x = var_821, y = var_822)[name = tensor("filled_window_15")]; + tensor var_824 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_824")]; + tensor var_825_dtype_0 = const()[name = tensor("op_825_dtype_0"), val = tensor("fp32")]; + tensor var_826 = const()[name = tensor("op_826"), val = tensor([1, 1, 1])]; + tensor var_825 = cast(dtype = var_825_dtype_0, x = var_824)[name = tensor("cast_144")]; + tensor use_roll_15 = reshape(shape = var_826, x = var_825)[name = tensor("use_roll_15")]; + tensor var_828 = sub(x = var_17, y = use_roll_15)[name = tensor("op_828")]; + tensor var_829 = mul(x = filled_window_15, y = var_828)[name = tensor("op_829")]; + tensor var_830 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_830")]; + tensor window_21 = add(x = var_829, y = var_830)[name = tensor("window_21")]; + tensor var_832 = const()[name = tensor("op_832"), val = tensor(0x1p+0)]; + tensor var_833 = add(x = clip_13, y = var_832)[name = tensor("op_833")]; + tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_833)[name = tensor("clip_15")]; + tensor var_836_begin_0 = const()[name = tensor("op_836_begin_0"), val = tensor([0, 2, 0])]; + tensor var_836_end_0 = const()[name = tensor("op_836_end_0"), val = tensor([1, 1, 256])]; + tensor var_836_end_mask_0 = const()[name = tensor("op_836_end_mask_0"), val = tensor([true, true, true])]; + tensor var_836 = slice_by_index(begin = var_836_begin_0, end = var_836_end_0, end_mask = var_836_end_mask_0, x = x_15)[name = tensor("op_836")]; + tensor var_839_begin_0 = const()[name = tensor("op_839_begin_0"), val = tensor([0, 1, 0])]; + tensor var_839_end_0 = const()[name = tensor("op_839_end_0"), val = tensor([1, 16, 256])]; + tensor var_839_end_mask_0 = const()[name = tensor("op_839_end_mask_0"), val = tensor([true, true, true])]; + tensor var_839 = slice_by_index(begin = var_839_begin_0, end = var_839_end_0, end_mask = var_839_end_mask_0, x = window_21)[name = tensor("op_839")]; tensor rolled_17_interleave_0 = const()[name = tensor("rolled_17_interleave_0"), val = tensor(false)]; - tensor rolled_17 = concat(axis = var_33, interleave = rolled_17_interleave_0, values = (var_1161, var_1158))[name = tensor("rolled_17")]; + tensor rolled_17 = concat(axis = var_33, interleave = rolled_17_interleave_0, values = (var_839, var_836))[name = tensor("rolled_17")]; tensor clip_16 = clip(alpha = var_21, beta = var_20, x = clip_15)[name = tensor("clip_16")]; - tensor var_1166 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_1166")]; - tensor var_1167_dtype_0 = const()[name = tensor("op_1167_dtype_0"), val = tensor("fp32")]; - tensor var_1168 = const()[name = tensor("op_1168"), val = tensor([1, 16, 1])]; - tensor var_1167 = cast(dtype = var_1167_dtype_0, x = var_1166)[name = tensor("cast_143")]; - tensor fill_mask_17 = reshape(shape = var_1168, x = var_1167)[name = tensor("fill_mask_17")]; + tensor var_844 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_844")]; + tensor var_845_dtype_0 = const()[name = tensor("op_845_dtype_0"), val = tensor("fp32")]; + tensor var_846 = const()[name = tensor("op_846"), val = tensor([1, 16, 1])]; + tensor var_845 = cast(dtype = var_845_dtype_0, x = var_844)[name = tensor("cast_143")]; + tensor fill_mask_17 = reshape(shape = var_846, x = var_845)[name = tensor("fill_mask_17")]; tensor x_expanded_17_reps_0 = const()[name = tensor("x_expanded_17_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_1158)[name = tensor("x_expanded_17")]; - tensor var_1172 = sub(x = var_17, y = fill_mask_17)[name = tensor("op_1172")]; - tensor var_1173 = mul(x = input_151, y = var_1172)[name = tensor("op_1173")]; - tensor var_1174 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_1174")]; - tensor filled_window_17 = add(x = var_1173, y = var_1174)[name = tensor("filled_window_17")]; - tensor var_1176 = greater_equal(x = clip_15, y = var_16)[name = tensor("op_1176")]; - tensor var_1177_dtype_0 = const()[name = tensor("op_1177_dtype_0"), val = tensor("fp32")]; - tensor var_1178 = const()[name = tensor("op_1178"), val = tensor([1, 1, 1])]; - tensor var_1177 = cast(dtype = var_1177_dtype_0, x = var_1176)[name = tensor("cast_142")]; - tensor use_roll_17 = reshape(shape = var_1178, x = var_1177)[name = tensor("use_roll_17")]; - tensor var_1180 = sub(x = var_17, y = use_roll_17)[name = tensor("op_1180")]; - tensor var_1181 = mul(x = filled_window_17, y = var_1180)[name = tensor("op_1181")]; - tensor var_1182 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_1182")]; - tensor input_161 = add(x = var_1181, y = var_1182)[name = tensor("input_161")]; - tensor var_1184 = const()[name = tensor("op_1184"), val = tensor(0x1p+0)]; - tensor var_1185 = add(x = clip_15, y = var_1184)[name = tensor("op_1185")]; - tensor clip_17 = clip(alpha = var_21, beta = var_16, x = var_1185)[name = tensor("clip_17")]; - tensor x_29_axes_0 = const()[name = tensor("x_29_axes_0"), val = tensor([-1])]; - tensor x_29 = layer_norm(axes = x_29_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_161)[name = tensor("x_29")]; - tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_59_pad_type_0 = const()[name = tensor("inputs_59_pad_type_0"), val = tensor("valid")]; - tensor inputs_59_strides_0 = const()[name = tensor("inputs_59_strides_0"), val = tensor([1])]; - tensor inputs_59_pad_0 = const()[name = tensor("inputs_59_pad_0"), val = tensor([0, 0])]; - tensor inputs_59_dilations_0 = const()[name = tensor("inputs_59_dilations_0"), val = tensor([1])]; - tensor inputs_59_groups_0 = const()[name = tensor("inputs_59_groups_0"), val = tensor(1)]; - tensor input_163 = transpose(perm = input_163_perm_0, x = x_29)[name = tensor("transpose_42")]; - tensor inputs_59 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_59_dilations_0, groups = inputs_59_groups_0, pad = inputs_59_pad_0, pad_type = inputs_59_pad_type_0, strides = inputs_59_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_163)[name = tensor("inputs_59")]; - tensor var_1206_split_sizes_0 = const()[name = tensor("op_1206_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1206_axis_0 = const()[name = tensor("op_1206_axis_0"), val = tensor(1)]; - tensor var_1206_0, tensor var_1206_1 = split(axis = var_1206_axis_0, split_sizes = var_1206_split_sizes_0, x = inputs_59)[name = tensor("op_1206")]; - tensor var_1208 = sigmoid(x = var_1206_1)[name = tensor("op_1208")]; - tensor inputs_61 = mul(x = var_1206_0, y = var_1208)[name = tensor("inputs_61")]; - tensor outputs_aug_17_pad_type_0 = const()[name = tensor("outputs_aug_17_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_17_pad_0 = const()[name = tensor("outputs_aug_17_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_17_groups_0 = const()[name = tensor("outputs_aug_17_groups_0"), val = tensor(256)]; - tensor outputs_aug_17_strides_0 = const()[name = tensor("outputs_aug_17_strides_0"), val = tensor([1])]; - tensor outputs_aug_17_dilations_0 = const()[name = tensor("outputs_aug_17_dilations_0"), val = tensor([1])]; - tensor outputs_aug_17 = conv(dilations = outputs_aug_17_dilations_0, groups = outputs_aug_17_groups_0, pad = outputs_aug_17_pad_0, pad_type = outputs_aug_17_pad_type_0, strides = outputs_aug_17_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_61)[name = tensor("outputs_aug_17")]; - tensor input_165_begin_0 = const()[name = tensor("input_165_begin_0"), val = tensor([0, 0, 0])]; - tensor input_165_end_0 = const()[name = tensor("input_165_end_0"), val = tensor([1, 256, 16])]; - tensor input_165_end_mask_0 = const()[name = tensor("input_165_end_mask_0"), val = tensor([true, true, false])]; - tensor input_165 = slice_by_index(begin = input_165_begin_0, end = input_165_end_0, end_mask = input_165_end_mask_0, x = outputs_aug_17)[name = tensor("input_165")]; - tensor inputs_63 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_165)[name = tensor("inputs_63")]; - tensor input_167 = silu(x = inputs_63)[name = tensor("input_167")]; - tensor input_169_pad_type_0 = const()[name = tensor("input_169_pad_type_0"), val = tensor("valid")]; - tensor input_169_strides_0 = const()[name = tensor("input_169_strides_0"), val = tensor([1])]; - tensor input_169_pad_0 = const()[name = tensor("input_169_pad_0"), val = tensor([0, 0])]; - tensor input_169_dilations_0 = const()[name = tensor("input_169_dilations_0"), val = tensor([1])]; - tensor input_169_groups_0 = const()[name = tensor("input_169_groups_0"), val = tensor(1)]; - tensor input_169 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_169_dilations_0, groups = input_169_groups_0, pad = input_169_pad_0, pad_type = input_169_pad_type_0, strides = input_169_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_167)[name = tensor("input_169")]; - tensor conv_out_17_perm_0 = const()[name = tensor("conv_out_17_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1239_begin_0 = const()[name = tensor("op_1239_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1239_end_0 = const()[name = tensor("op_1239_end_0"), val = tensor([1, 16, 256])]; - tensor var_1239_end_mask_0 = const()[name = tensor("op_1239_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_17 = transpose(perm = conv_out_17_perm_0, x = input_169)[name = tensor("transpose_41")]; - tensor var_1239 = slice_by_index(begin = var_1239_begin_0, end = var_1239_end_0, end_mask = var_1239_end_mask_0, x = conv_out_17)[name = tensor("op_1239")]; - tensor var_1242_interleave_0 = const()[name = tensor("op_1242_interleave_0"), val = tensor(false)]; - tensor var_1242 = concat(axis = var_33, interleave = var_1242_interleave_0, values = (var_1071, var_1155, var_1239))[name = tensor("op_1242")]; - tensor input_171 = add(x = x_23, y = var_1242)[name = tensor("input_171")]; - tensor input_173_axes_0 = const()[name = tensor("input_173_axes_0"), val = tensor([-1])]; - tensor input_173 = layer_norm(axes = input_173_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_171)[name = tensor("input_173")]; - tensor inputs_65 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_173)[name = tensor("linear_26")]; - tensor input_175 = silu(x = inputs_65)[name = tensor("input_175")]; - tensor input_179 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_175)[name = tensor("linear_27")]; - tensor var_1265 = const()[name = tensor("op_1265"), val = tensor(0x1p-1)]; - tensor var_1266 = mul(x = input_179, y = var_1265)[name = tensor("op_1266")]; - tensor input_181 = add(x = var_1266, y = input_171)[name = tensor("input_181")]; - tensor input_183_axes_0 = const()[name = tensor("input_183_axes_0"), val = tensor([-1])]; - tensor input_183 = layer_norm(axes = input_183_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_181)[name = tensor("input_183")]; - tensor input_185_axes_0 = const()[name = tensor("input_185_axes_0"), val = tensor([-1])]; - tensor input_185 = layer_norm(axes = input_185_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_183)[name = tensor("input_185")]; - tensor inputs_67 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_185)[name = tensor("linear_28")]; - tensor input_187 = silu(x = inputs_67)[name = tensor("input_187")]; - tensor input_191 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_187)[name = tensor("linear_29")]; - tensor var_1295 = const()[name = tensor("op_1295"), val = tensor(0x1p-1)]; - tensor var_1296 = mul(x = input_191, y = var_1295)[name = tensor("op_1296")]; - tensor input_193 = add(x = var_1296, y = input_183)[name = tensor("input_193")]; - tensor x_31_axes_0 = const()[name = tensor("x_31_axes_0"), val = tensor([-1])]; - tensor x_31 = layer_norm(axes = x_31_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_193)[name = tensor("x_31")]; + tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_836)[name = tensor("x_expanded_17")]; + tensor var_850 = sub(x = var_17, y = fill_mask_17)[name = tensor("op_850")]; + tensor var_851 = mul(x = window_21, y = var_850)[name = tensor("op_851")]; + tensor var_852 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_852")]; + tensor filled_window_17 = add(x = var_851, y = var_852)[name = tensor("filled_window_17")]; + tensor var_854 = greater_equal(x = clip_15, y = var_16)[name = tensor("op_854")]; + tensor var_855_dtype_0 = const()[name = tensor("op_855_dtype_0"), val = tensor("fp32")]; + tensor var_856 = const()[name = tensor("op_856"), val = tensor([1, 1, 1])]; + tensor var_855 = cast(dtype = var_855_dtype_0, x = var_854)[name = tensor("cast_142")]; + tensor use_roll_17 = reshape(shape = var_856, x = var_855)[name = tensor("use_roll_17")]; + tensor var_858 = sub(x = var_17, y = use_roll_17)[name = tensor("op_858")]; + tensor var_859 = mul(x = filled_window_17, y = var_858)[name = tensor("op_859")]; + tensor var_860 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_860")]; + tensor window_23 = add(x = var_859, y = var_860)[name = tensor("window_23")]; + tensor var_862 = const()[name = tensor("op_862"), val = tensor(0x1p+0)]; + tensor var_863 = add(x = clip_15, y = var_862)[name = tensor("op_863")]; + tensor clip_17 = clip(alpha = var_21, beta = var_16, x = var_863)[name = tensor("clip_17")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_30, interleave = input_101_interleave_0, values = (window_19, window_21, window_23))[name = tensor("input_101")]; + tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; + tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; + tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_23_pad_type_0 = const()[name = tensor("inputs_23_pad_type_0"), val = tensor("valid")]; + tensor inputs_23_strides_0 = const()[name = tensor("inputs_23_strides_0"), val = tensor([1])]; + tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; + tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; + tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; + tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; + tensor var_886_split_sizes_0 = const()[name = tensor("op_886_split_sizes_0"), val = tensor([256, 256])]; + tensor var_886_axis_0 = const()[name = tensor("op_886_axis_0"), val = tensor(1)]; + tensor var_886_0, tensor var_886_1 = split(axis = var_886_axis_0, split_sizes = var_886_split_sizes_0, x = inputs_23)[name = tensor("op_886")]; + tensor var_888 = sigmoid(x = var_886_1)[name = tensor("op_888")]; + tensor inputs_25 = mul(x = var_886_0, y = var_888)[name = tensor("inputs_25")]; + tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; + tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; + tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; + tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_25)[name = tensor("outputs_aug_5")]; + tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; + tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([3, 256, 16])]; + tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; + tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_5)[name = tensor("input_105")]; + tensor inputs_27 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_105)[name = tensor("inputs_27")]; + tensor input_107 = silu(x = inputs_27)[name = tensor("input_107")]; + tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; + tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; + tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; + tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; + tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; + tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; + tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; + tensor var_919_begin_0 = const()[name = tensor("op_919_begin_0"), val = tensor([0, -1, 0])]; + tensor var_919_end_0 = const()[name = tensor("op_919_end_0"), val = tensor([3, 16, 256])]; + tensor var_919_end_mask_0 = const()[name = tensor("op_919_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_919 = slice_by_index(begin = var_919_begin_0, end = var_919_end_0, end_mask = var_919_end_mask_0, x = conv_out_5)[name = tensor("op_919")]; + tensor var_921_perm_0 = const()[name = tensor("op_921_perm_0"), val = tensor([1, 0, 2])]; + tensor var_921 = transpose(perm = var_921_perm_0, x = var_919)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_921)[name = tensor("input_111")]; + tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; + tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; + tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; + tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; + tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; + tensor var_944 = const()[name = tensor("op_944"), val = tensor(0x1p-1)]; + tensor var_945 = mul(x = input_119, y = var_944)[name = tensor("op_945")]; + tensor input_121 = add(x = var_945, y = input_111)[name = tensor("input_121")]; + tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; + tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; + tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; + tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; + tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; + tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; + tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; + tensor var_974 = const()[name = tensor("op_974"), val = tensor(0x1p-1)]; + tensor var_975 = mul(x = input_131, y = var_974)[name = tensor("op_975")]; + tensor input_133 = add(x = var_975, y = input_123)[name = tensor("input_133")]; + tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; + tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; tensor prev_kv_7_end_0 = const()[name = tensor("prev_kv_7_end_0"), val = tensor([4, 1, 4, 64, 64])]; tensor prev_kv_7_end_mask_0 = const()[name = tensor("prev_kv_7_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1203,309 +976,233 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_1310 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_31)[name = tensor("linear_30")]; - tensor var_1311 = const()[name = tensor("op_1311"), val = tensor([1, 3, 4, 64])]; - tensor var_1312 = reshape(shape = var_1311, x = var_1310)[name = tensor("op_1312")]; + tensor var_989 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_990 = const()[name = tensor("op_990"), val = tensor([1, 3, 4, 64])]; + tensor var_991 = reshape(shape = var_990, x = var_989)[name = tensor("op_991")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1316 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_31)[name = tensor("linear_31")]; - tensor var_1317 = const()[name = tensor("op_1317"), val = tensor(0x1p-3)]; - tensor var_1318 = mul(x = var_1316, y = var_1317)[name = tensor("op_1318")]; - tensor var_1319 = const()[name = tensor("op_1319"), val = tensor([1, 3, 4, 64])]; - tensor var_1320 = reshape(shape = var_1319, x = var_1318)[name = tensor("op_1320")]; + tensor var_995 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_996 = const()[name = tensor("op_996"), val = tensor(0x1p-3)]; + tensor var_997 = mul(x = var_995, y = var_996)[name = tensor("op_997")]; + tensor var_998 = const()[name = tensor("op_998"), val = tensor([1, 3, 4, 64])]; + tensor var_999 = reshape(shape = var_998, x = var_997)[name = tensor("op_999")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1324 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_31)[name = tensor("linear_32")]; - tensor var_1325 = const()[name = tensor("op_1325"), val = tensor([1, 3, 4, 64])]; - tensor var_1326 = reshape(shape = var_1325, x = var_1324)[name = tensor("op_1326")]; + tensor var_1003 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_1004 = const()[name = tensor("op_1004"), val = tensor([1, 3, 4, 64])]; + tensor var_1005 = reshape(shape = var_1004, x = var_1003)[name = tensor("op_1005")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_197 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_31)[name = tensor("linear_33")]; + tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; tensor s_t_7 = add(x = prev_scale_7, y = encoder__t_index)[name = tensor("s_t_7")]; tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_1320)[name = tensor("transpose_39")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_1312)[name = tensor("transpose_40")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_999)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_991)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_1336 = const()[name = tensor("op_1336"), val = tensor([3, 1])]; - tensor var_1337 = reshape(shape = var_1336, x = sqrt_s_t_7)[name = tensor("op_1337")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_1337)[name = tensor("M_7")]; - tensor var_1339 = mul(x = qk_7, y = M_7)[name = tensor("op_1339")]; + tensor var_1015 = const()[name = tensor("op_1015"), val = tensor([3, 1])]; + tensor var_1016 = reshape(shape = var_1015, x = sqrt_s_t_7)[name = tensor("op_1016")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_1016)[name = tensor("M_7")]; + tensor var_1018 = mul(x = qk_7, y = M_7)[name = tensor("op_1018")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_1326)[name = tensor("transpose_38")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1339, y = v_7)[name = tensor("inner_7")]; - tensor var_1341_transpose_x_0 = const()[name = tensor("op_1341_transpose_x_0"), val = tensor(false)]; - tensor var_1341_transpose_y_0 = const()[name = tensor("op_1341_transpose_y_0"), val = tensor(false)]; - tensor var_1341 = matmul(transpose_x = var_1341_transpose_x_0, transpose_y = var_1341_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1341")]; - tensor var_1342 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1342")]; - tensor var_1343 = const()[name = tensor("op_1343"), val = tensor([1, 1, 3, 1])]; - tensor var_1344 = reshape(shape = var_1343, x = var_1342)[name = tensor("op_1344")]; - tensor cross_7 = mul(x = var_1341, y = var_1344)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_1005)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1018, y = v_7)[name = tensor("inner_7")]; + tensor var_1020_transpose_x_0 = const()[name = tensor("op_1020_transpose_x_0"), val = tensor(false)]; + tensor var_1020_transpose_y_0 = const()[name = tensor("op_1020_transpose_y_0"), val = tensor(false)]; + tensor var_1020 = matmul(transpose_x = var_1020_transpose_x_0, transpose_y = var_1020_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1020")]; + tensor var_1021 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1021")]; + tensor var_1022 = const()[name = tensor("op_1022"), val = tensor([1, 1, 3, 1])]; + tensor var_1023 = reshape(shape = var_1022, x = var_1021)[name = tensor("op_1023")]; + tensor cross_7 = mul(x = var_1020, y = var_1023)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_1347 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1347")]; - tensor var_1349_transpose_x_1 = const()[name = tensor("op_1349_transpose_x_1"), val = tensor(true)]; - tensor var_1349_transpose_y_1 = const()[name = tensor("op_1349_transpose_y_1"), val = tensor(false)]; - tensor var_1349 = matmul(transpose_x = var_1349_transpose_x_1, transpose_y = var_1349_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1349")]; - tensor new_kv_unnorm_7 = add(x = var_1347, y = var_1349)[name = tensor("new_kv_unnorm_7")]; - tensor var_1351 = const()[name = tensor("op_1351"), val = tensor(0x1.8p+1)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_1351)[name = tensor("new_scale_7")]; - tensor var_1353 = sqrt(x = new_scale_7)[name = tensor("op_1353")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1353)[name = tensor("nkv_1")]; - tensor var_1355_perm_0 = const()[name = tensor("op_1355_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1026 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1026")]; + tensor var_1028_transpose_x_1 = const()[name = tensor("op_1028_transpose_x_1"), val = tensor(true)]; + tensor var_1028_transpose_y_1 = const()[name = tensor("op_1028_transpose_y_1"), val = tensor(false)]; + tensor var_1028 = matmul(transpose_x = var_1028_transpose_x_1, transpose_y = var_1028_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1028")]; + tensor new_kv_unnorm_7 = add(x = var_1026, y = var_1028)[name = tensor("new_kv_unnorm_7")]; + tensor var_1030 = const()[name = tensor("op_1030"), val = tensor(0x1.8p+1)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_1030)[name = tensor("new_scale_7")]; + tensor var_1032 = sqrt(x = new_scale_7)[name = tensor("op_1032")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1032)[name = tensor("nkv_1")]; + tensor var_1034_perm_0 = const()[name = tensor("op_1034_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_1355 = transpose(perm = var_1355_perm_0, x = out_19)[name = tensor("transpose_37")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1355)[name = tensor("out_21")]; - tensor var_1359 = const()[name = tensor("op_1359"), val = tensor([1, 3, 256])]; - tensor out_23 = reshape(shape = var_1359, x = out_21)[name = tensor("out_23")]; - tensor var_1361 = silu(x = input_197)[name = tensor("op_1361")]; - tensor input_199 = mul(x = var_1361, y = out_23)[name = tensor("input_199")]; - tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_199)[name = tensor("linear_34")]; - tensor x_33 = add(x = input_193, y = ret_out_7)[name = tensor("x_33")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor var_1034 = transpose(perm = var_1034_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1034)[name = tensor("out_21")]; + tensor var_1038 = const()[name = tensor("op_1038"), val = tensor([1, 3, 256])]; + tensor out_23 = reshape(shape = var_1038, x = out_21)[name = tensor("out_23")]; + tensor var_1040 = silu(x = input_137)[name = tensor("op_1040")]; + tensor input_139 = mul(x = var_1040, y = out_23)[name = tensor("input_139")]; + tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; + tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; + tensor window_25_begin_0 = const()[name = tensor("window_25_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_25_end_0 = const()[name = tensor("window_25_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_25_end_mask_0 = const()[name = tensor("window_25_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_25_squeeze_mask_0 = const()[name = tensor("window_25_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_25 = slice_by_index(begin = window_25_begin_0, end = window_25_end_0, end_mask = window_25_end_mask_0, squeeze_mask = window_25_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_25")]; tensor filled_19_begin_0 = const()[name = tensor("filled_19_begin_0"), val = tensor([3, 0])]; tensor filled_19_end_0 = const()[name = tensor("filled_19_end_0"), val = tensor([4, 1])]; tensor filled_19_end_mask_0 = const()[name = tensor("filled_19_end_mask_0"), val = tensor([false, true])]; tensor filled_19_squeeze_mask_0 = const()[name = tensor("filled_19_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_19 = slice_by_index(begin = filled_19_begin_0, end = filled_19_end_0, end_mask = filled_19_end_mask_0, squeeze_mask = filled_19_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_19")]; - tensor var_1370_begin_0 = const()[name = tensor("op_1370_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1370_end_0 = const()[name = tensor("op_1370_end_0"), val = tensor([1, 1, 256])]; - tensor var_1370_end_mask_0 = const()[name = tensor("op_1370_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1370 = slice_by_index(begin = var_1370_begin_0, end = var_1370_end_0, end_mask = var_1370_end_mask_0, x = x_33)[name = tensor("op_1370")]; - tensor var_1373_begin_0 = const()[name = tensor("op_1373_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1373_end_0 = const()[name = tensor("op_1373_end_0"), val = tensor([1, 16, 256])]; - tensor var_1373_end_mask_0 = const()[name = tensor("op_1373_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1373 = slice_by_index(begin = var_1373_begin_0, end = var_1373_end_0, end_mask = var_1373_end_mask_0, x = window)[name = tensor("op_1373")]; + tensor var_1049_begin_0 = const()[name = tensor("op_1049_begin_0"), val = tensor([0, 0, 0])]; + tensor var_1049_end_0 = const()[name = tensor("op_1049_end_0"), val = tensor([1, 1, 256])]; + tensor var_1049_end_mask_0 = const()[name = tensor("op_1049_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1049 = slice_by_index(begin = var_1049_begin_0, end = var_1049_end_0, end_mask = var_1049_end_mask_0, x = x_21)[name = tensor("op_1049")]; + tensor var_1052_begin_0 = const()[name = tensor("op_1052_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1052_end_0 = const()[name = tensor("op_1052_end_0"), val = tensor([1, 16, 256])]; + tensor var_1052_end_mask_0 = const()[name = tensor("op_1052_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1052 = slice_by_index(begin = var_1052_begin_0, end = var_1052_end_0, end_mask = var_1052_end_mask_0, x = window_25)[name = tensor("op_1052")]; tensor rolled_19_interleave_0 = const()[name = tensor("rolled_19_interleave_0"), val = tensor(false)]; - tensor rolled_19 = concat(axis = var_33, interleave = rolled_19_interleave_0, values = (var_1373, var_1370))[name = tensor("rolled_19")]; + tensor rolled_19 = concat(axis = var_33, interleave = rolled_19_interleave_0, values = (var_1052, var_1049))[name = tensor("rolled_19")]; tensor clip_18 = clip(alpha = var_21, beta = var_20, x = filled_19)[name = tensor("clip_18")]; - tensor var_1378 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_1378")]; - tensor var_1379_dtype_0 = const()[name = tensor("op_1379_dtype_0"), val = tensor("fp32")]; - tensor var_1380 = const()[name = tensor("op_1380"), val = tensor([1, 16, 1])]; - tensor var_1379 = cast(dtype = var_1379_dtype_0, x = var_1378)[name = tensor("cast_141")]; - tensor fill_mask_19 = reshape(shape = var_1380, x = var_1379)[name = tensor("fill_mask_19")]; + tensor var_1057 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_1057")]; + tensor var_1058_dtype_0 = const()[name = tensor("op_1058_dtype_0"), val = tensor("fp32")]; + tensor var_1059 = const()[name = tensor("op_1059"), val = tensor([1, 16, 1])]; + tensor var_1058 = cast(dtype = var_1058_dtype_0, x = var_1057)[name = tensor("cast_141")]; + tensor fill_mask_19 = reshape(shape = var_1059, x = var_1058)[name = tensor("fill_mask_19")]; tensor x_expanded_19_reps_0 = const()[name = tensor("x_expanded_19_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_1370)[name = tensor("x_expanded_19")]; - tensor var_1384 = sub(x = var_17, y = fill_mask_19)[name = tensor("op_1384")]; - tensor var_1385 = mul(x = window, y = var_1384)[name = tensor("op_1385")]; - tensor var_1386 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_1386")]; - tensor filled_window_19 = add(x = var_1385, y = var_1386)[name = tensor("filled_window_19")]; - tensor var_1388 = greater_equal(x = filled_19, y = var_16)[name = tensor("op_1388")]; - tensor var_1389_dtype_0 = const()[name = tensor("op_1389_dtype_0"), val = tensor("fp32")]; - tensor var_1390 = const()[name = tensor("op_1390"), val = tensor([1, 1, 1])]; - tensor var_1389 = cast(dtype = var_1389_dtype_0, x = var_1388)[name = tensor("cast_140")]; - tensor use_roll_19 = reshape(shape = var_1390, x = var_1389)[name = tensor("use_roll_19")]; - tensor var_1392 = sub(x = var_17, y = use_roll_19)[name = tensor("op_1392")]; - tensor var_1393 = mul(x = filled_window_19, y = var_1392)[name = tensor("op_1393")]; - tensor var_1394 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_1394")]; - tensor input_201 = add(x = var_1393, y = var_1394)[name = tensor("input_201")]; - tensor var_1396 = const()[name = tensor("op_1396"), val = tensor(0x1p+0)]; - tensor var_1397 = add(x = filled_19, y = var_1396)[name = tensor("op_1397")]; - tensor clip_19 = clip(alpha = var_21, beta = var_16, x = var_1397)[name = tensor("clip_19")]; - tensor x_35_axes_0 = const()[name = tensor("x_35_axes_0"), val = tensor([-1])]; - tensor x_35 = layer_norm(axes = x_35_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_201)[name = tensor("x_35")]; - tensor input_203_perm_0 = const()[name = tensor("input_203_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_69_pad_type_0 = const()[name = tensor("inputs_69_pad_type_0"), val = tensor("valid")]; - tensor inputs_69_strides_0 = const()[name = tensor("inputs_69_strides_0"), val = tensor([1])]; - tensor inputs_69_pad_0 = const()[name = tensor("inputs_69_pad_0"), val = tensor([0, 0])]; - tensor inputs_69_dilations_0 = const()[name = tensor("inputs_69_dilations_0"), val = tensor([1])]; - tensor inputs_69_groups_0 = const()[name = tensor("inputs_69_groups_0"), val = tensor(1)]; - tensor input_203 = transpose(perm = input_203_perm_0, x = x_35)[name = tensor("transpose_36")]; - tensor inputs_69 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_69_dilations_0, groups = inputs_69_groups_0, pad = inputs_69_pad_0, pad_type = inputs_69_pad_type_0, strides = inputs_69_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_203)[name = tensor("inputs_69")]; - tensor var_1418_split_sizes_0 = const()[name = tensor("op_1418_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1418_axis_0 = const()[name = tensor("op_1418_axis_0"), val = tensor(1)]; - tensor var_1418_0, tensor var_1418_1 = split(axis = var_1418_axis_0, split_sizes = var_1418_split_sizes_0, x = inputs_69)[name = tensor("op_1418")]; - tensor var_1420 = sigmoid(x = var_1418_1)[name = tensor("op_1420")]; - tensor inputs_71 = mul(x = var_1418_0, y = var_1420)[name = tensor("inputs_71")]; - tensor outputs_aug_19_pad_type_0 = const()[name = tensor("outputs_aug_19_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_19_pad_0 = const()[name = tensor("outputs_aug_19_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_19_groups_0 = const()[name = tensor("outputs_aug_19_groups_0"), val = tensor(256)]; - tensor outputs_aug_19_strides_0 = const()[name = tensor("outputs_aug_19_strides_0"), val = tensor([1])]; - tensor outputs_aug_19_dilations_0 = const()[name = tensor("outputs_aug_19_dilations_0"), val = tensor([1])]; - tensor outputs_aug_19 = conv(dilations = outputs_aug_19_dilations_0, groups = outputs_aug_19_groups_0, pad = outputs_aug_19_pad_0, pad_type = outputs_aug_19_pad_type_0, strides = outputs_aug_19_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_71)[name = tensor("outputs_aug_19")]; - tensor input_205_begin_0 = const()[name = tensor("input_205_begin_0"), val = tensor([0, 0, 0])]; - tensor input_205_end_0 = const()[name = tensor("input_205_end_0"), val = tensor([1, 256, 16])]; - tensor input_205_end_mask_0 = const()[name = tensor("input_205_end_mask_0"), val = tensor([true, true, false])]; - tensor input_205 = slice_by_index(begin = input_205_begin_0, end = input_205_end_0, end_mask = input_205_end_mask_0, x = outputs_aug_19)[name = tensor("input_205")]; - tensor inputs_73 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_205)[name = tensor("inputs_73")]; - tensor input_207 = silu(x = inputs_73)[name = tensor("input_207")]; - tensor input_209_pad_type_0 = const()[name = tensor("input_209_pad_type_0"), val = tensor("valid")]; - tensor input_209_strides_0 = const()[name = tensor("input_209_strides_0"), val = tensor([1])]; - tensor input_209_pad_0 = const()[name = tensor("input_209_pad_0"), val = tensor([0, 0])]; - tensor input_209_dilations_0 = const()[name = tensor("input_209_dilations_0"), val = tensor([1])]; - tensor input_209_groups_0 = const()[name = tensor("input_209_groups_0"), val = tensor(1)]; - tensor input_209 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_209_dilations_0, groups = input_209_groups_0, pad = input_209_pad_0, pad_type = input_209_pad_type_0, strides = input_209_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_207)[name = tensor("input_209")]; - tensor conv_out_19_perm_0 = const()[name = tensor("conv_out_19_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1451_begin_0 = const()[name = tensor("op_1451_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1451_end_0 = const()[name = tensor("op_1451_end_0"), val = tensor([1, 16, 256])]; - tensor var_1451_end_mask_0 = const()[name = tensor("op_1451_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_19 = transpose(perm = conv_out_19_perm_0, x = input_209)[name = tensor("transpose_35")]; - tensor var_1451 = slice_by_index(begin = var_1451_begin_0, end = var_1451_end_0, end_mask = var_1451_end_mask_0, x = conv_out_19)[name = tensor("op_1451")]; - tensor var_1454_begin_0 = const()[name = tensor("op_1454_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1454_end_0 = const()[name = tensor("op_1454_end_0"), val = tensor([1, 2, 256])]; - tensor var_1454_end_mask_0 = const()[name = tensor("op_1454_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1454 = slice_by_index(begin = var_1454_begin_0, end = var_1454_end_0, end_mask = var_1454_end_mask_0, x = x_33)[name = tensor("op_1454")]; - tensor var_1457_begin_0 = const()[name = tensor("op_1457_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1457_end_0 = const()[name = tensor("op_1457_end_0"), val = tensor([1, 16, 256])]; - tensor var_1457_end_mask_0 = const()[name = tensor("op_1457_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1457 = slice_by_index(begin = var_1457_begin_0, end = var_1457_end_0, end_mask = var_1457_end_mask_0, x = input_201)[name = tensor("op_1457")]; + tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_1049)[name = tensor("x_expanded_19")]; + tensor var_1063 = sub(x = var_17, y = fill_mask_19)[name = tensor("op_1063")]; + tensor var_1064 = mul(x = window_25, y = var_1063)[name = tensor("op_1064")]; + tensor var_1065 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_1065")]; + tensor filled_window_19 = add(x = var_1064, y = var_1065)[name = tensor("filled_window_19")]; + tensor var_1067 = greater_equal(x = filled_19, y = var_16)[name = tensor("op_1067")]; + tensor var_1068_dtype_0 = const()[name = tensor("op_1068_dtype_0"), val = tensor("fp32")]; + tensor var_1069 = const()[name = tensor("op_1069"), val = tensor([1, 1, 1])]; + tensor var_1068 = cast(dtype = var_1068_dtype_0, x = var_1067)[name = tensor("cast_140")]; + tensor use_roll_19 = reshape(shape = var_1069, x = var_1068)[name = tensor("use_roll_19")]; + tensor var_1071 = sub(x = var_17, y = use_roll_19)[name = tensor("op_1071")]; + tensor var_1072 = mul(x = filled_window_19, y = var_1071)[name = tensor("op_1072")]; + tensor var_1073 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_1073")]; + tensor window_27 = add(x = var_1072, y = var_1073)[name = tensor("window_27")]; + tensor var_1075 = const()[name = tensor("op_1075"), val = tensor(0x1p+0)]; + tensor var_1076 = add(x = filled_19, y = var_1075)[name = tensor("op_1076")]; + tensor clip_19 = clip(alpha = var_21, beta = var_16, x = var_1076)[name = tensor("clip_19")]; + tensor var_1079_begin_0 = const()[name = tensor("op_1079_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1079_end_0 = const()[name = tensor("op_1079_end_0"), val = tensor([1, 2, 256])]; + tensor var_1079_end_mask_0 = const()[name = tensor("op_1079_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1079 = slice_by_index(begin = var_1079_begin_0, end = var_1079_end_0, end_mask = var_1079_end_mask_0, x = x_21)[name = tensor("op_1079")]; + tensor var_1082_begin_0 = const()[name = tensor("op_1082_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1082_end_0 = const()[name = tensor("op_1082_end_0"), val = tensor([1, 16, 256])]; + tensor var_1082_end_mask_0 = const()[name = tensor("op_1082_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1082 = slice_by_index(begin = var_1082_begin_0, end = var_1082_end_0, end_mask = var_1082_end_mask_0, x = window_27)[name = tensor("op_1082")]; tensor rolled_21_interleave_0 = const()[name = tensor("rolled_21_interleave_0"), val = tensor(false)]; - tensor rolled_21 = concat(axis = var_33, interleave = rolled_21_interleave_0, values = (var_1457, var_1454))[name = tensor("rolled_21")]; + tensor rolled_21 = concat(axis = var_33, interleave = rolled_21_interleave_0, values = (var_1082, var_1079))[name = tensor("rolled_21")]; tensor clip_20 = clip(alpha = var_21, beta = var_20, x = clip_19)[name = tensor("clip_20")]; - tensor var_1462 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_1462")]; - tensor var_1463_dtype_0 = const()[name = tensor("op_1463_dtype_0"), val = tensor("fp32")]; - tensor var_1464 = const()[name = tensor("op_1464"), val = tensor([1, 16, 1])]; - tensor var_1463 = cast(dtype = var_1463_dtype_0, x = var_1462)[name = tensor("cast_139")]; - tensor fill_mask_21 = reshape(shape = var_1464, x = var_1463)[name = tensor("fill_mask_21")]; + tensor var_1087 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_1087")]; + tensor var_1088_dtype_0 = const()[name = tensor("op_1088_dtype_0"), val = tensor("fp32")]; + tensor var_1089 = const()[name = tensor("op_1089"), val = tensor([1, 16, 1])]; + tensor var_1088 = cast(dtype = var_1088_dtype_0, x = var_1087)[name = tensor("cast_139")]; + tensor fill_mask_21 = reshape(shape = var_1089, x = var_1088)[name = tensor("fill_mask_21")]; tensor x_expanded_21_reps_0 = const()[name = tensor("x_expanded_21_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_1454)[name = tensor("x_expanded_21")]; - tensor var_1468 = sub(x = var_17, y = fill_mask_21)[name = tensor("op_1468")]; - tensor var_1469 = mul(x = input_201, y = var_1468)[name = tensor("op_1469")]; - tensor var_1470 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_1470")]; - tensor filled_window_21 = add(x = var_1469, y = var_1470)[name = tensor("filled_window_21")]; - tensor var_1472 = greater_equal(x = clip_19, y = var_16)[name = tensor("op_1472")]; - tensor var_1473_dtype_0 = const()[name = tensor("op_1473_dtype_0"), val = tensor("fp32")]; - tensor var_1474 = const()[name = tensor("op_1474"), val = tensor([1, 1, 1])]; - tensor var_1473 = cast(dtype = var_1473_dtype_0, x = var_1472)[name = tensor("cast_138")]; - tensor use_roll_21 = reshape(shape = var_1474, x = var_1473)[name = tensor("use_roll_21")]; - tensor var_1476 = sub(x = var_17, y = use_roll_21)[name = tensor("op_1476")]; - tensor var_1477 = mul(x = filled_window_21, y = var_1476)[name = tensor("op_1477")]; - tensor var_1478 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_1478")]; - tensor input_211 = add(x = var_1477, y = var_1478)[name = tensor("input_211")]; - tensor var_1480 = const()[name = tensor("op_1480"), val = tensor(0x1p+0)]; - tensor var_1481 = add(x = clip_19, y = var_1480)[name = tensor("op_1481")]; - tensor clip_21 = clip(alpha = var_21, beta = var_16, x = var_1481)[name = tensor("clip_21")]; - tensor x_37_axes_0 = const()[name = tensor("x_37_axes_0"), val = tensor([-1])]; - tensor x_37 = layer_norm(axes = x_37_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_211)[name = tensor("x_37")]; - tensor input_213_perm_0 = const()[name = tensor("input_213_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_75_pad_type_0 = const()[name = tensor("inputs_75_pad_type_0"), val = tensor("valid")]; - tensor inputs_75_strides_0 = const()[name = tensor("inputs_75_strides_0"), val = tensor([1])]; - tensor inputs_75_pad_0 = const()[name = tensor("inputs_75_pad_0"), val = tensor([0, 0])]; - tensor inputs_75_dilations_0 = const()[name = tensor("inputs_75_dilations_0"), val = tensor([1])]; - tensor inputs_75_groups_0 = const()[name = tensor("inputs_75_groups_0"), val = tensor(1)]; - tensor input_213 = transpose(perm = input_213_perm_0, x = x_37)[name = tensor("transpose_34")]; - tensor inputs_75 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_75_dilations_0, groups = inputs_75_groups_0, pad = inputs_75_pad_0, pad_type = inputs_75_pad_type_0, strides = inputs_75_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_213)[name = tensor("inputs_75")]; - tensor var_1502_split_sizes_0 = const()[name = tensor("op_1502_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1502_axis_0 = const()[name = tensor("op_1502_axis_0"), val = tensor(1)]; - tensor var_1502_0, tensor var_1502_1 = split(axis = var_1502_axis_0, split_sizes = var_1502_split_sizes_0, x = inputs_75)[name = tensor("op_1502")]; - tensor var_1504 = sigmoid(x = var_1502_1)[name = tensor("op_1504")]; - tensor inputs_77 = mul(x = var_1502_0, y = var_1504)[name = tensor("inputs_77")]; - tensor outputs_aug_21_pad_type_0 = const()[name = tensor("outputs_aug_21_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_21_pad_0 = const()[name = tensor("outputs_aug_21_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_21_groups_0 = const()[name = tensor("outputs_aug_21_groups_0"), val = tensor(256)]; - tensor outputs_aug_21_strides_0 = const()[name = tensor("outputs_aug_21_strides_0"), val = tensor([1])]; - tensor outputs_aug_21_dilations_0 = const()[name = tensor("outputs_aug_21_dilations_0"), val = tensor([1])]; - tensor outputs_aug_21 = conv(dilations = outputs_aug_21_dilations_0, groups = outputs_aug_21_groups_0, pad = outputs_aug_21_pad_0, pad_type = outputs_aug_21_pad_type_0, strides = outputs_aug_21_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_77)[name = tensor("outputs_aug_21")]; - tensor input_215_begin_0 = const()[name = tensor("input_215_begin_0"), val = tensor([0, 0, 0])]; - tensor input_215_end_0 = const()[name = tensor("input_215_end_0"), val = tensor([1, 256, 16])]; - tensor input_215_end_mask_0 = const()[name = tensor("input_215_end_mask_0"), val = tensor([true, true, false])]; - tensor input_215 = slice_by_index(begin = input_215_begin_0, end = input_215_end_0, end_mask = input_215_end_mask_0, x = outputs_aug_21)[name = tensor("input_215")]; - tensor inputs_79 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_215)[name = tensor("inputs_79")]; - tensor input_217 = silu(x = inputs_79)[name = tensor("input_217")]; - tensor input_219_pad_type_0 = const()[name = tensor("input_219_pad_type_0"), val = tensor("valid")]; - tensor input_219_strides_0 = const()[name = tensor("input_219_strides_0"), val = tensor([1])]; - tensor input_219_pad_0 = const()[name = tensor("input_219_pad_0"), val = tensor([0, 0])]; - tensor input_219_dilations_0 = const()[name = tensor("input_219_dilations_0"), val = tensor([1])]; - tensor input_219_groups_0 = const()[name = tensor("input_219_groups_0"), val = tensor(1)]; - tensor input_219 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_219_dilations_0, groups = input_219_groups_0, pad = input_219_pad_0, pad_type = input_219_pad_type_0, strides = input_219_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_217)[name = tensor("input_219")]; - tensor conv_out_21_perm_0 = const()[name = tensor("conv_out_21_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1535_begin_0 = const()[name = tensor("op_1535_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1535_end_0 = const()[name = tensor("op_1535_end_0"), val = tensor([1, 16, 256])]; - tensor var_1535_end_mask_0 = const()[name = tensor("op_1535_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_21 = transpose(perm = conv_out_21_perm_0, x = input_219)[name = tensor("transpose_33")]; - tensor var_1535 = slice_by_index(begin = var_1535_begin_0, end = var_1535_end_0, end_mask = var_1535_end_mask_0, x = conv_out_21)[name = tensor("op_1535")]; - tensor var_1538_begin_0 = const()[name = tensor("op_1538_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1538_end_0 = const()[name = tensor("op_1538_end_0"), val = tensor([1, 1, 256])]; - tensor var_1538_end_mask_0 = const()[name = tensor("op_1538_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1538 = slice_by_index(begin = var_1538_begin_0, end = var_1538_end_0, end_mask = var_1538_end_mask_0, x = x_33)[name = tensor("op_1538")]; - tensor var_1541_begin_0 = const()[name = tensor("op_1541_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1541_end_0 = const()[name = tensor("op_1541_end_0"), val = tensor([1, 16, 256])]; - tensor var_1541_end_mask_0 = const()[name = tensor("op_1541_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1541 = slice_by_index(begin = var_1541_begin_0, end = var_1541_end_0, end_mask = var_1541_end_mask_0, x = input_211)[name = tensor("op_1541")]; + tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_1079)[name = tensor("x_expanded_21")]; + tensor var_1093 = sub(x = var_17, y = fill_mask_21)[name = tensor("op_1093")]; + tensor var_1094 = mul(x = window_27, y = var_1093)[name = tensor("op_1094")]; + tensor var_1095 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_1095")]; + tensor filled_window_21 = add(x = var_1094, y = var_1095)[name = tensor("filled_window_21")]; + tensor var_1097 = greater_equal(x = clip_19, y = var_16)[name = tensor("op_1097")]; + tensor var_1098_dtype_0 = const()[name = tensor("op_1098_dtype_0"), val = tensor("fp32")]; + tensor var_1099 = const()[name = tensor("op_1099"), val = tensor([1, 1, 1])]; + tensor var_1098 = cast(dtype = var_1098_dtype_0, x = var_1097)[name = tensor("cast_138")]; + tensor use_roll_21 = reshape(shape = var_1099, x = var_1098)[name = tensor("use_roll_21")]; + tensor var_1101 = sub(x = var_17, y = use_roll_21)[name = tensor("op_1101")]; + tensor var_1102 = mul(x = filled_window_21, y = var_1101)[name = tensor("op_1102")]; + tensor var_1103 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_1103")]; + tensor window_29 = add(x = var_1102, y = var_1103)[name = tensor("window_29")]; + tensor var_1105 = const()[name = tensor("op_1105"), val = tensor(0x1p+0)]; + tensor var_1106 = add(x = clip_19, y = var_1105)[name = tensor("op_1106")]; + tensor clip_21 = clip(alpha = var_21, beta = var_16, x = var_1106)[name = tensor("clip_21")]; + tensor var_1109_begin_0 = const()[name = tensor("op_1109_begin_0"), val = tensor([0, 2, 0])]; + tensor var_1109_end_0 = const()[name = tensor("op_1109_end_0"), val = tensor([1, 1, 256])]; + tensor var_1109_end_mask_0 = const()[name = tensor("op_1109_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1109 = slice_by_index(begin = var_1109_begin_0, end = var_1109_end_0, end_mask = var_1109_end_mask_0, x = x_21)[name = tensor("op_1109")]; + tensor var_1112_begin_0 = const()[name = tensor("op_1112_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1112_end_0 = const()[name = tensor("op_1112_end_0"), val = tensor([1, 16, 256])]; + tensor var_1112_end_mask_0 = const()[name = tensor("op_1112_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1112 = slice_by_index(begin = var_1112_begin_0, end = var_1112_end_0, end_mask = var_1112_end_mask_0, x = window_29)[name = tensor("op_1112")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1541, var_1538))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1112, var_1109))[name = tensor("rolled")]; tensor clip_22 = clip(alpha = var_21, beta = var_20, x = clip_21)[name = tensor("clip_22")]; - tensor var_1546 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_1546")]; - tensor var_1547_dtype_0 = const()[name = tensor("op_1547_dtype_0"), val = tensor("fp32")]; - tensor var_1548 = const()[name = tensor("op_1548"), val = tensor([1, 16, 1])]; - tensor var_1547 = cast(dtype = var_1547_dtype_0, x = var_1546)[name = tensor("cast_137")]; - tensor fill_mask = reshape(shape = var_1548, x = var_1547)[name = tensor("fill_mask")]; + tensor var_1117 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_1117")]; + tensor var_1118_dtype_0 = const()[name = tensor("op_1118_dtype_0"), val = tensor("fp32")]; + tensor var_1119 = const()[name = tensor("op_1119"), val = tensor([1, 16, 1])]; + tensor var_1118 = cast(dtype = var_1118_dtype_0, x = var_1117)[name = tensor("cast_137")]; + tensor fill_mask = reshape(shape = var_1119, x = var_1118)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1538)[name = tensor("x_expanded")]; - tensor var_1552 = sub(x = var_17, y = fill_mask)[name = tensor("op_1552")]; - tensor var_1553 = mul(x = input_211, y = var_1552)[name = tensor("op_1553")]; - tensor var_1554 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1554")]; - tensor filled_window = add(x = var_1553, y = var_1554)[name = tensor("filled_window")]; - tensor var_1556 = greater_equal(x = clip_21, y = var_16)[name = tensor("op_1556")]; - tensor var_1557_dtype_0 = const()[name = tensor("op_1557_dtype_0"), val = tensor("fp32")]; - tensor var_1558 = const()[name = tensor("op_1558"), val = tensor([1, 1, 1])]; - tensor var_1557 = cast(dtype = var_1557_dtype_0, x = var_1556)[name = tensor("cast_136")]; - tensor use_roll = reshape(shape = var_1558, x = var_1557)[name = tensor("use_roll")]; - tensor var_1560 = sub(x = var_17, y = use_roll)[name = tensor("op_1560")]; - tensor var_1561 = mul(x = filled_window, y = var_1560)[name = tensor("op_1561")]; - tensor var_1562 = mul(x = rolled, y = use_roll)[name = tensor("op_1562")]; - tensor input_221 = add(x = var_1561, y = var_1562)[name = tensor("input_221")]; - tensor var_1564 = const()[name = tensor("op_1564"), val = tensor(0x1p+0)]; - tensor var_1565 = add(x = clip_21, y = var_1564)[name = tensor("op_1565")]; - tensor clip_23 = clip(alpha = var_21, beta = var_16, x = var_1565)[name = tensor("clip_23")]; - tensor x_39_axes_0 = const()[name = tensor("x_39_axes_0"), val = tensor([-1])]; - tensor x_39 = layer_norm(axes = x_39_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_221)[name = tensor("x_39")]; - tensor input_223_perm_0 = const()[name = tensor("input_223_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_81_pad_type_0 = const()[name = tensor("inputs_81_pad_type_0"), val = tensor("valid")]; - tensor inputs_81_strides_0 = const()[name = tensor("inputs_81_strides_0"), val = tensor([1])]; - tensor inputs_81_pad_0 = const()[name = tensor("inputs_81_pad_0"), val = tensor([0, 0])]; - tensor inputs_81_dilations_0 = const()[name = tensor("inputs_81_dilations_0"), val = tensor([1])]; - tensor inputs_81_groups_0 = const()[name = tensor("inputs_81_groups_0"), val = tensor(1)]; - tensor input_223 = transpose(perm = input_223_perm_0, x = x_39)[name = tensor("transpose_32")]; - tensor inputs_81 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_81_dilations_0, groups = inputs_81_groups_0, pad = inputs_81_pad_0, pad_type = inputs_81_pad_type_0, strides = inputs_81_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_223)[name = tensor("inputs_81")]; - tensor var_1586_split_sizes_0 = const()[name = tensor("op_1586_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1586_axis_0 = const()[name = tensor("op_1586_axis_0"), val = tensor(1)]; - tensor var_1586_0, tensor var_1586_1 = split(axis = var_1586_axis_0, split_sizes = var_1586_split_sizes_0, x = inputs_81)[name = tensor("op_1586")]; - tensor var_1588 = sigmoid(x = var_1586_1)[name = tensor("op_1588")]; - tensor inputs_83 = mul(x = var_1586_0, y = var_1588)[name = tensor("inputs_83")]; + tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1109)[name = tensor("x_expanded")]; + tensor var_1123 = sub(x = var_17, y = fill_mask)[name = tensor("op_1123")]; + tensor var_1124 = mul(x = window_29, y = var_1123)[name = tensor("op_1124")]; + tensor var_1125 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1125")]; + tensor filled_window = add(x = var_1124, y = var_1125)[name = tensor("filled_window")]; + tensor var_1127 = greater_equal(x = clip_21, y = var_16)[name = tensor("op_1127")]; + tensor var_1128_dtype_0 = const()[name = tensor("op_1128_dtype_0"), val = tensor("fp32")]; + tensor var_1129 = const()[name = tensor("op_1129"), val = tensor([1, 1, 1])]; + tensor var_1128 = cast(dtype = var_1128_dtype_0, x = var_1127)[name = tensor("cast_136")]; + tensor use_roll = reshape(shape = var_1129, x = var_1128)[name = tensor("use_roll")]; + tensor var_1131 = sub(x = var_17, y = use_roll)[name = tensor("op_1131")]; + tensor var_1132 = mul(x = filled_window, y = var_1131)[name = tensor("op_1132")]; + tensor var_1133 = mul(x = rolled, y = use_roll)[name = tensor("op_1133")]; + tensor window = add(x = var_1132, y = var_1133)[name = tensor("window")]; + tensor var_1135 = const()[name = tensor("op_1135"), val = tensor(0x1p+0)]; + tensor var_1136 = add(x = clip_21, y = var_1135)[name = tensor("op_1136")]; + tensor clip_23 = clip(alpha = var_21, beta = var_16, x = var_1136)[name = tensor("clip_23")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_30, interleave = input_141_interleave_0, values = (window_27, window_29, window))[name = tensor("input_141")]; + tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; + tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; + tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_33_pad_type_0 = const()[name = tensor("inputs_33_pad_type_0"), val = tensor("valid")]; + tensor inputs_33_strides_0 = const()[name = tensor("inputs_33_strides_0"), val = tensor([1])]; + tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; + tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; + tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; + tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; + tensor var_1159_split_sizes_0 = const()[name = tensor("op_1159_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1159_axis_0 = const()[name = tensor("op_1159_axis_0"), val = tensor(1)]; + tensor var_1159_0, tensor var_1159_1 = split(axis = var_1159_axis_0, split_sizes = var_1159_split_sizes_0, x = inputs_33)[name = tensor("op_1159")]; + tensor var_1161 = sigmoid(x = var_1159_1)[name = tensor("op_1161")]; + tensor inputs_35 = mul(x = var_1159_0, y = var_1161)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; tensor outputs_aug_strides_0 = const()[name = tensor("outputs_aug_strides_0"), val = tensor([1])]; tensor outputs_aug_dilations_0 = const()[name = tensor("outputs_aug_dilations_0"), val = tensor([1])]; - tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_83)[name = tensor("outputs_aug")]; - tensor input_225_begin_0 = const()[name = tensor("input_225_begin_0"), val = tensor([0, 0, 0])]; - tensor input_225_end_0 = const()[name = tensor("input_225_end_0"), val = tensor([1, 256, 16])]; - tensor input_225_end_mask_0 = const()[name = tensor("input_225_end_mask_0"), val = tensor([true, true, false])]; - tensor input_225 = slice_by_index(begin = input_225_begin_0, end = input_225_end_0, end_mask = input_225_end_mask_0, x = outputs_aug)[name = tensor("input_225")]; - tensor inputs_85 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_225)[name = tensor("inputs_85")]; - tensor input_227 = silu(x = inputs_85)[name = tensor("input_227")]; - tensor input_229_pad_type_0 = const()[name = tensor("input_229_pad_type_0"), val = tensor("valid")]; - tensor input_229_strides_0 = const()[name = tensor("input_229_strides_0"), val = tensor([1])]; - tensor input_229_pad_0 = const()[name = tensor("input_229_pad_0"), val = tensor([0, 0])]; - tensor input_229_dilations_0 = const()[name = tensor("input_229_dilations_0"), val = tensor([1])]; - tensor input_229_groups_0 = const()[name = tensor("input_229_groups_0"), val = tensor(1)]; - tensor input_229 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_229_dilations_0, groups = input_229_groups_0, pad = input_229_pad_0, pad_type = input_229_pad_type_0, strides = input_229_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_227)[name = tensor("input_229")]; - tensor conv_out_23_perm_0 = const()[name = tensor("conv_out_23_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1619_begin_0 = const()[name = tensor("op_1619_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1619_end_0 = const()[name = tensor("op_1619_end_0"), val = tensor([1, 16, 256])]; - tensor var_1619_end_mask_0 = const()[name = tensor("op_1619_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_23 = transpose(perm = conv_out_23_perm_0, x = input_229)[name = tensor("transpose_31")]; - tensor var_1619 = slice_by_index(begin = var_1619_begin_0, end = var_1619_end_0, end_mask = var_1619_end_mask_0, x = conv_out_23)[name = tensor("op_1619")]; - tensor var_1622_interleave_0 = const()[name = tensor("op_1622_interleave_0"), val = tensor(false)]; - tensor var_1622 = concat(axis = var_33, interleave = var_1622_interleave_0, values = (var_1451, var_1535, var_1619))[name = tensor("op_1622")]; - tensor input_231 = add(x = x_33, y = var_1622)[name = tensor("input_231")]; - tensor input_233_axes_0 = const()[name = tensor("input_233_axes_0"), val = tensor([-1])]; - tensor input_233 = layer_norm(axes = input_233_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_231)[name = tensor("input_233")]; - tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_233)[name = tensor("linear_35")]; - tensor input_235 = silu(x = inputs)[name = tensor("input_235")]; - tensor input_239 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_235)[name = tensor("linear_36")]; - tensor var_1645 = const()[name = tensor("op_1645"), val = tensor(0x1p-1)]; - tensor var_1646 = mul(x = input_239, y = var_1645)[name = tensor("op_1646")]; - tensor input_241 = add(x = var_1646, y = input_231)[name = tensor("input_241")]; - tensor x_41_axes_0 = const()[name = tensor("x_41_axes_0"), val = tensor([-1])]; - tensor x_41 = layer_norm(axes = x_41_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_241)[name = tensor("x_41")]; + tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_35)[name = tensor("outputs_aug")]; + tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; + tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([3, 256, 16])]; + tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; + tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug)[name = tensor("input_145")]; + tensor inputs_37 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_145)[name = tensor("inputs_37")]; + tensor input_147 = silu(x = inputs_37)[name = tensor("input_147")]; + tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; + tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; + tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; + tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; + tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; + tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; + tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1192_begin_0 = const()[name = tensor("op_1192_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1192_end_0 = const()[name = tensor("op_1192_end_0"), val = tensor([3, 16, 256])]; + tensor var_1192_end_mask_0 = const()[name = tensor("op_1192_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_1192 = slice_by_index(begin = var_1192_begin_0, end = var_1192_end_0, end_mask = var_1192_end_mask_0, x = conv_out_7)[name = tensor("op_1192")]; + tensor var_1194_perm_0 = const()[name = tensor("op_1194_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1194 = transpose(perm = var_1194_perm_0, x = var_1192)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_1194)[name = tensor("input_151")]; + tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; + tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; + tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; + tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; + tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; + tensor var_1217 = const()[name = tensor("op_1217"), val = tensor(0x1p-1)]; + tensor var_1218 = mul(x = input_159, y = var_1217)[name = tensor("op_1218")]; + tensor input_161 = add(x = var_1218, y = input_151)[name = tensor("input_161")]; + tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; + tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; tensor cat_interleave_0 = const()[name = tensor("cat_interleave_0"), val = tensor(false)]; - tensor x_ct = transpose(perm = x_ct_perm_0, x = x_41)[name = tensor("transpose_30")]; + tensor x_ct = transpose(perm = x_ct_perm_0, x = x_25)[name = tensor("transpose_30")]; tensor cat = concat(axis = var_27, interleave = cat_interleave_0, values = (cnn_window, x_ct))[name = tensor("cat")]; tensor conv_out_pad_type_0 = const()[name = tensor("conv_out_pad_type_0"), val = tensor("valid")]; tensor conv_out_strides_0 = const()[name = tensor("conv_out_strides_0"), val = tensor([1])]; @@ -1513,44 +1210,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_1664_begin_0 = const()[name = tensor("op_1664_begin_0"), val = tensor([0, 0, 3])]; - tensor var_1664_end_0 = const()[name = tensor("op_1664_end_0"), val = tensor([1, 256, 21])]; - tensor var_1664_end_mask_0 = const()[name = tensor("op_1664_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_1664_begin_0, end = var_1664_end_0, end_mask = var_1664_end_mask_0, x = cat)[name = tensor("op_1664")]; - tensor input_243_perm_0 = const()[name = tensor("input_243_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1666 = const()[name = tensor("op_1666"), val = tensor([-1])]; - tensor input_243 = transpose(perm = input_243_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_1667 = reduce_l2_norm(axes = var_1666, keep_dims = var_36, x = input_243)[name = tensor("op_1667")]; - tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; - tensor clip_24 = clip(alpha = var_12, beta = const_20, x = var_1667)[name = tensor("clip_24")]; - tensor emb = real_div(x = input_243, y = clip_24)[name = tensor("emb")]; - tensor var_1671_axis_0 = const()[name = tensor("op_1671_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_1671_axis_0, values = (var_214, var_594, var_974, nkv_1))[name = tensor("op_1671")]; - tensor var_1673_axis_0 = const()[name = tensor("op_1673_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_1673_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1673")]; - tensor var_1675_axis_0 = const()[name = tensor("op_1675_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_1675_axis_0, values = (input_41, input_101, input_161, input_221))[name = tensor("op_1675")]; - tensor var_1677_axis_0 = const()[name = tensor("op_1677_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_1677_axis_0, values = (clip_5, clip_11, clip_17, clip_23))[name = tensor("op_1677")]; - tensor var_1687 = const()[name = tensor("op_1687"), val = tensor(0x1.5798eep-27)]; - tensor var_1693 = const()[name = tensor("op_1693"), val = tensor(0x1.4f8b58p-17)]; - tensor var_1695 = const()[name = tensor("op_1695"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_1696 = const()[name = tensor("op_1696"), val = tensor(true)]; - tensor var_1698 = const()[name = tensor("op_1698"), val = tensor(0x1p+0)]; - tensor var_1702 = const()[name = tensor("op_1702"), val = tensor(-1)]; - tensor var_1707 = const()[name = tensor("op_1707"), val = tensor(0)]; + tensor var_1236_begin_0 = const()[name = tensor("op_1236_begin_0"), val = tensor([0, 0, 3])]; + tensor var_1236_end_0 = const()[name = tensor("op_1236_end_0"), val = tensor([1, 256, 21])]; + tensor var_1236_end_mask_0 = const()[name = tensor("op_1236_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_1236_begin_0, end = var_1236_end_0, end_mask = var_1236_end_mask_0, x = cat)[name = tensor("op_1236")]; + tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1238 = const()[name = tensor("op_1238"), val = tensor([-1])]; + tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; + tensor var_1239 = reduce_l2_norm(axes = var_1238, keep_dims = var_36, x = input_163)[name = tensor("op_1239")]; + tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; + tensor clip_24 = clip(alpha = var_12, beta = const_12, x = var_1239)[name = tensor("clip_24")]; + tensor emb = real_div(x = input_163, y = clip_24)[name = tensor("emb")]; + tensor var_1243_axis_0 = const()[name = tensor("op_1243_axis_0"), val = tensor(0)]; + tensor enc_kv_new = stack(axis = var_1243_axis_0, values = (var_214, var_487, var_760, nkv_1))[name = tensor("op_1243")]; + tensor var_1245_axis_0 = const()[name = tensor("op_1245_axis_0"), val = tensor(0)]; + tensor enc_scale_new = stack(axis = var_1245_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1245")]; + tensor var_1247_axis_0 = const()[name = tensor("op_1247_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1247_axis_0, values = (window_7, window_15, window_23, window))[name = tensor("op_1247")]; + tensor var_1249_axis_0 = const()[name = tensor("op_1249_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1249_axis_0, values = (clip_5, clip_11, clip_17, clip_23))[name = tensor("op_1249")]; + tensor var_1259 = const()[name = tensor("op_1259"), val = tensor(0x1.5798eep-27)]; + tensor var_1264 = const()[name = tensor("op_1264"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1266 = const()[name = tensor("op_1266"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1267 = const()[name = tensor("op_1267"), val = tensor(true)]; + tensor var_1269 = const()[name = tensor("op_1269"), val = tensor(0x1p+0)]; + tensor var_1273 = const()[name = tensor("op_1273"), val = tensor(-1)]; + tensor var_1279 = const()[name = tensor("op_1279"), val = tensor(0)]; tensor pos = const()[name = tensor("pos"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395456)))]; - tensor var_1769_axes_0 = const()[name = tensor("op_1769_axes_0"), val = tensor([2])]; - tensor var_1769 = expand_dims(axes = var_1769_axes_0, x = emb)[name = tensor("op_1769")]; - tensor var_1770 = const()[name = tensor("op_1770"), val = tensor([1, 1, 9, 1])]; - tensor emb_exp = tile(reps = var_1770, x = var_1769)[name = tensor("emb_exp")]; - tensor input_245_interleave_0 = const()[name = tensor("input_245_interleave_0"), val = tensor(false)]; - tensor input_245 = concat(axis = var_1702, interleave = input_245_interleave_0, values = (emb_exp, pos))[name = tensor("input_245")]; - tensor x_43 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_245)[name = tensor("linear_37")]; - tensor var_1777_perm_0 = const()[name = tensor("op_1777_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1781 = const()[name = tensor("op_1781"), val = tensor([9, 3, 256])]; - tensor var_1777 = transpose(perm = var_1777_perm_0, x = x_43)[name = tensor("transpose_28")]; - tensor x_45 = reshape(shape = var_1781, x = var_1777)[name = tensor("x_45")]; + tensor var_1341_axes_0 = const()[name = tensor("op_1341_axes_0"), val = tensor([2])]; + tensor var_1341 = expand_dims(axes = var_1341_axes_0, x = emb)[name = tensor("op_1341")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 9, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1341)[name = tensor("emb_exp")]; + tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; + tensor input_165 = concat(axis = var_1273, interleave = input_165_interleave_0, values = (emb_exp, pos))[name = tensor("input_165")]; + tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; + tensor var_1349_perm_0 = const()[name = tensor("op_1349_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1353 = const()[name = tensor("op_1353"), val = tensor([9, 3, 256])]; + tensor var_1349 = transpose(perm = var_1349_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1353, x = var_1349)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 9, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1561,132 +1258,132 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_1789 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_45)[name = tensor("linear_38")]; - tensor var_1790 = const()[name = tensor("op_1790"), val = tensor([9, 3, 4, 64])]; - tensor var_1791 = reshape(shape = var_1790, x = var_1789)[name = tensor("op_1791")]; + tensor var_1361 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1362 = const()[name = tensor("op_1362"), val = tensor([9, 3, 4, 64])]; + tensor var_1363 = reshape(shape = var_1362, x = var_1361)[name = tensor("op_1363")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1795 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_45)[name = tensor("linear_39")]; - tensor var_1796 = const()[name = tensor("op_1796"), val = tensor(0x1p-3)]; - tensor var_1797 = mul(x = var_1795, y = var_1796)[name = tensor("op_1797")]; - tensor var_1798 = const()[name = tensor("op_1798"), val = tensor([9, 3, 4, 64])]; - tensor var_1799 = reshape(shape = var_1798, x = var_1797)[name = tensor("op_1799")]; + tensor var_1367 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1368 = const()[name = tensor("op_1368"), val = tensor(0x1p-3)]; + tensor var_1369 = mul(x = var_1367, y = var_1368)[name = tensor("op_1369")]; + tensor var_1370 = const()[name = tensor("op_1370"), val = tensor([9, 3, 4, 64])]; + tensor var_1371 = reshape(shape = var_1370, x = var_1369)[name = tensor("op_1371")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1803 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_45)[name = tensor("linear_40")]; - tensor var_1804 = const()[name = tensor("op_1804"), val = tensor([9, 3, 4, 64])]; - tensor var_1805 = reshape(shape = var_1804, x = var_1803)[name = tensor("op_1805")]; + tensor var_1375 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1376 = const()[name = tensor("op_1376"), val = tensor([9, 3, 4, 64])]; + tensor var_1377 = reshape(shape = var_1376, x = var_1375)[name = tensor("op_1377")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_249 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_45)[name = tensor("linear_41")]; + tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_1707, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1279, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; - tensor const_28 = const()[name = tensor("const_28"), val = tensor(0x1.fffffep+127)]; - tensor clip_25 = clip(alpha = var_1698, beta = const_28, x = s_t_9)[name = tensor("clip_25")]; + tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; + tensor clip_25 = clip(alpha = var_1269, beta = const_20, x = s_t_9)[name = tensor("clip_25")]; tensor sqrt_s_t_9 = sqrt(x = clip_25)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_1799)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_1791)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1371)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1363)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_1817 = const()[name = tensor("op_1817"), val = tensor([1, 3])]; - tensor var_1818 = reshape(shape = var_1817, x = valid_mask)[name = tensor("op_1818")]; - tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1818)[name = tensor("causal_with_valid_1")]; - tensor var_1820 = const()[name = tensor("op_1820"), val = tensor([3, 1])]; - tensor var_1821 = reshape(shape = var_1820, x = sqrt_s_t_9)[name = tensor("op_1821")]; - tensor M_9 = real_div(x = causal_with_valid_1, y = var_1821)[name = tensor("M_9")]; - tensor var_1823 = mul(x = qk_9, y = M_9)[name = tensor("op_1823")]; + tensor var_1389 = const()[name = tensor("op_1389"), val = tensor([1, 3])]; + tensor var_1390 = reshape(shape = var_1389, x = valid_mask)[name = tensor("op_1390")]; + tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1390)[name = tensor("causal_with_valid_1")]; + tensor var_1392 = const()[name = tensor("op_1392"), val = tensor([3, 1])]; + tensor var_1393 = reshape(shape = var_1392, x = sqrt_s_t_9)[name = tensor("op_1393")]; + tensor M_9 = real_div(x = causal_with_valid_1, y = var_1393)[name = tensor("M_9")]; + tensor var_1395 = mul(x = qk_9, y = M_9)[name = tensor("op_1395")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_1805)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1823, y = v_9)[name = tensor("inner_9")]; - tensor var_1825_transpose_x_0 = const()[name = tensor("op_1825_transpose_x_0"), val = tensor(false)]; - tensor var_1825_transpose_y_0 = const()[name = tensor("op_1825_transpose_y_0"), val = tensor(false)]; - tensor var_1825 = matmul(transpose_x = var_1825_transpose_x_0, transpose_y = var_1825_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1825")]; - tensor var_1826 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1826")]; - tensor var_1827 = const()[name = tensor("op_1827"), val = tensor([1, 1, 3, 1])]; - tensor var_1828 = reshape(shape = var_1827, x = var_1826)[name = tensor("op_1828")]; - tensor cross_9 = mul(x = var_1825, y = var_1828)[name = tensor("cross_9")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1377)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1395, y = v_9)[name = tensor("inner_9")]; + tensor var_1397_transpose_x_0 = const()[name = tensor("op_1397_transpose_x_0"), val = tensor(false)]; + tensor var_1397_transpose_y_0 = const()[name = tensor("op_1397_transpose_y_0"), val = tensor(false)]; + tensor var_1397 = matmul(transpose_x = var_1397_transpose_x_0, transpose_y = var_1397_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1397")]; + tensor var_1398 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1398")]; + tensor var_1399 = const()[name = tensor("op_1399"), val = tensor([1, 1, 3, 1])]; + tensor var_1400 = reshape(shape = var_1399, x = var_1398)[name = tensor("op_1400")]; + tensor cross_9 = mul(x = var_1397, y = var_1400)[name = tensor("cross_9")]; tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; - tensor var_1831 = const()[name = tensor("op_1831"), val = tensor([1, 1, 3, 1])]; - tensor var_1832 = reshape(shape = var_1831, x = valid_mask)[name = tensor("op_1832")]; - tensor v_masked_1 = mul(x = v_9, y = var_1832)[name = tensor("v_masked_1")]; - tensor var_1834 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1834")]; - tensor var_1836_transpose_x_1 = const()[name = tensor("op_1836_transpose_x_1"), val = tensor(true)]; - tensor var_1836_transpose_y_1 = const()[name = tensor("op_1836_transpose_y_1"), val = tensor(false)]; - tensor var_1836 = matmul(transpose_x = var_1836_transpose_x_1, transpose_y = var_1836_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1836")]; - tensor new_kv_unnorm_9 = add(x = var_1834, y = var_1836)[name = tensor("new_kv_unnorm_9")]; - tensor var_1838_keep_dims_0 = const()[name = tensor("op_1838_keep_dims_0"), val = tensor(false)]; - tensor var_1838 = reduce_sum(keep_dims = var_1838_keep_dims_0, x = valid_mask)[name = tensor("op_1838")]; - tensor var_1839 = const()[name = tensor("op_1839"), val = tensor([1])]; - tensor var_1840 = reshape(shape = var_1839, x = var_1838)[name = tensor("op_1840")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_1840)[name = tensor("new_scale_9")]; - tensor const_29 = const()[name = tensor("const_29"), val = tensor(0x1.fffffep+127)]; - tensor clip_26 = clip(alpha = var_1698, beta = const_29, x = new_scale_9)[name = tensor("clip_26")]; + tensor var_1403 = const()[name = tensor("op_1403"), val = tensor([1, 1, 3, 1])]; + tensor var_1404 = reshape(shape = var_1403, x = valid_mask)[name = tensor("op_1404")]; + tensor v_masked_1 = mul(x = v_9, y = var_1404)[name = tensor("v_masked_1")]; + tensor var_1406 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1406")]; + tensor var_1408_transpose_x_1 = const()[name = tensor("op_1408_transpose_x_1"), val = tensor(true)]; + tensor var_1408_transpose_y_1 = const()[name = tensor("op_1408_transpose_y_1"), val = tensor(false)]; + tensor var_1408 = matmul(transpose_x = var_1408_transpose_x_1, transpose_y = var_1408_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1408")]; + tensor new_kv_unnorm_9 = add(x = var_1406, y = var_1408)[name = tensor("new_kv_unnorm_9")]; + tensor var_1410_keep_dims_0 = const()[name = tensor("op_1410_keep_dims_0"), val = tensor(false)]; + tensor var_1410 = reduce_sum(keep_dims = var_1410_keep_dims_0, x = valid_mask)[name = tensor("op_1410")]; + tensor var_1411 = const()[name = tensor("op_1411"), val = tensor([1])]; + tensor var_1412 = reshape(shape = var_1411, x = var_1410)[name = tensor("op_1412")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1412)[name = tensor("new_scale_9")]; + tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; + tensor clip_26 = clip(alpha = var_1269, beta = const_21, x = new_scale_9)[name = tensor("clip_26")]; tensor sqrt_new_scale_1 = sqrt(x = clip_26)[name = tensor("sqrt_new_scale_1")]; - tensor var_1844 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1844")]; - tensor var_1845_perm_0 = const()[name = tensor("op_1845_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1416 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1416")]; + tensor var_1417_perm_0 = const()[name = tensor("op_1417_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_1845 = transpose(perm = var_1845_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1695, x = var_1845)[name = tensor("out_27")]; - tensor var_1849 = const()[name = tensor("op_1849"), val = tensor([9, 3, 256])]; - tensor out_29 = reshape(shape = var_1849, x = out_27)[name = tensor("out_29")]; - tensor var_1851 = silu(x = input_249)[name = tensor("op_1851")]; - tensor input_251 = mul(x = var_1851, y = out_29)[name = tensor("input_251")]; - tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_251)[name = tensor("linear_42")]; - tensor input_253 = add(x = x_45, y = ret_out_9)[name = tensor("input_253")]; + tensor var_1417 = transpose(perm = var_1417_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1266, x = var_1417)[name = tensor("out_27")]; + tensor var_1421 = const()[name = tensor("op_1421"), val = tensor([9, 3, 256])]; + tensor out_29 = reshape(shape = var_1421, x = out_27)[name = tensor("out_29")]; + tensor var_1423 = silu(x = input_169)[name = tensor("op_1423")]; + tensor input_171 = mul(x = var_1423, y = out_29)[name = tensor("input_171")]; + tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; + tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1693, gamma = decoder_norm11_0_weight, x = input_253)[name = tensor("xt_1")]; - tensor var_1861 = const()[name = tensor("op_1861"), val = tensor([1, 9, 3, 256])]; - tensor var_1862 = reshape(shape = var_1861, x = xt_1)[name = tensor("op_1862")]; - tensor var_1863_perm_0 = const()[name = tensor("op_1863_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1866 = const()[name = tensor("op_1866"), val = tensor([3, 9, 256])]; - tensor var_1863 = transpose(perm = var_1863_perm_0, x = var_1862)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_1866, x = var_1863)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1264, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1433 = const()[name = tensor("op_1433"), val = tensor([1, 9, 3, 256])]; + tensor var_1434 = reshape(shape = var_1433, x = xt_1)[name = tensor("op_1434")]; + tensor var_1435_perm_0 = const()[name = tensor("op_1435_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1438 = const()[name = tensor("op_1438"), val = tensor([3, 9, 256])]; + tensor var_1435 = transpose(perm = var_1435_perm_0, x = var_1434)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1438, x = var_1435)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_1889 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_12 = const()[name = tensor("concat_12"), val = tensor([9, 3, 3, 256])]; - tensor var_1891 = reshape(shape = concat_12, x = var_1889)[name = tensor("op_1891")]; - tensor var_1892_axes_0 = const()[name = tensor("op_1892_axes_0"), val = tensor([0])]; - tensor var_1892 = expand_dims(axes = var_1892_axes_0, x = var_1891)[name = tensor("op_1892")]; - tensor var_1893_perm_0 = const()[name = tensor("op_1893_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1894_axes_0 = const()[name = tensor("op_1894_axes_0"), val = tensor([-2])]; - tensor var_1893 = transpose(perm = var_1893_perm_0, x = var_1892)[name = tensor("transpose_21")]; - tensor var_1894 = squeeze(axes = var_1894_axes_0, x = var_1893)[name = tensor("op_1894")]; + tensor var_1461 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_13 = const()[name = tensor("concat_13"), val = tensor([9, 3, 3, 256])]; + tensor var_1463 = reshape(shape = concat_13, x = var_1461)[name = tensor("op_1463")]; + tensor var_1464_axes_0 = const()[name = tensor("op_1464_axes_0"), val = tensor([0])]; + tensor var_1464 = expand_dims(axes = var_1464_axes_0, x = var_1463)[name = tensor("op_1464")]; + tensor var_1465_perm_0 = const()[name = tensor("op_1465_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1466_axes_0 = const()[name = tensor("op_1466_axes_0"), val = tensor([-2])]; + tensor var_1465 = transpose(perm = var_1465_perm_0, x = var_1464)[name = tensor("transpose_21")]; + tensor var_1466 = squeeze(axes = var_1466_axes_0, x = var_1465)[name = tensor("op_1466")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 9, 3, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1894)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1466)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 9, 3, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1894)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1466)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 9, 3, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1894)[name = tensor("v_11")]; - tensor var_1902 = const()[name = tensor("op_1902"), val = tensor([9, 12, 64])]; - tensor var_1903 = reshape(shape = var_1902, x = q_11)[name = tensor("op_1903")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1466)[name = tensor("v_11")]; + tensor var_1474 = const()[name = tensor("op_1474"), val = tensor([9, 12, 64])]; + tensor var_1475 = reshape(shape = var_1474, x = q_11)[name = tensor("op_1475")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1909 = const()[name = tensor("op_1909"), val = tensor([9, 12, 64])]; - tensor var_1910 = reshape(shape = var_1909, x = k_11)[name = tensor("op_1910")]; + tensor var_1481 = const()[name = tensor("op_1481"), val = tensor([9, 12, 64])]; + tensor var_1482 = reshape(shape = var_1481, x = k_11)[name = tensor("op_1482")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1916 = const()[name = tensor("op_1916"), val = tensor([9, 12, 64])]; - tensor var_1917 = reshape(shape = var_1916, x = v_11)[name = tensor("op_1917")]; + tensor var_1488 = const()[name = tensor("op_1488"), val = tensor([9, 12, 64])]; + tensor var_1489 = reshape(shape = var_1488, x = v_11)[name = tensor("op_1489")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1920 = const()[name = tensor("op_1920"), val = tensor([3, 4, 9, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_1903)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_1920, x = q_13)[name = tensor("q_15")]; - tensor var_1922 = const()[name = tensor("op_1922"), val = tensor([3, 4, 9, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_1910)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_1922, x = k_13)[name = tensor("k_15")]; - tensor var_1924 = const()[name = tensor("op_1924"), val = tensor([3, 4, 9, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_1917)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_1924, x = v_13)[name = tensor("v_15")]; + tensor var_1492 = const()[name = tensor("op_1492"), val = tensor([3, 4, 9, 64])]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1475)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1492, x = q_13)[name = tensor("q_15")]; + tensor var_1494 = const()[name = tensor("op_1494"), val = tensor([3, 4, 9, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1482)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1494, x = k_13)[name = tensor("k_15")]; + tensor var_1496 = const()[name = tensor("op_1496"), val = tensor([3, 4, 9, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1489)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1496, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -1697,30 +1394,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_1927 = const()[name = tensor("op_1927"), val = tensor([2, 0, 1, 3])]; - tensor var_1932 = const()[name = tensor("op_1932"), val = tensor([27, 256])]; - tensor var_1928 = transpose(perm = var_1927, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_1932, x = var_1928)[name = tensor("attn_output_3")]; + tensor var_1499 = const()[name = tensor("op_1499"), val = tensor([2, 0, 1, 3])]; + tensor var_1504 = const()[name = tensor("op_1504"), val = tensor([27, 256])]; + tensor var_1500 = transpose(perm = var_1499, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1504, x = var_1500)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_1936 = const()[name = tensor("op_1936"), val = tensor([9, 3, 256])]; - tensor attn_output_7 = reshape(shape = var_1936, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1508 = const()[name = tensor("op_1508"), val = tensor([9, 3, 256])]; + tensor attn_output_7 = reshape(shape = var_1508, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; - tensor input_255 = add(x = query_1, y = sa2_out_1)[name = tensor("input_255")]; - tensor input_257_axes_0 = const()[name = tensor("input_257_axes_0"), val = tensor([-1])]; - tensor input_257 = layer_norm(axes = input_257_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1693, gamma = decoder_norm21_0_weight, x = input_255)[name = tensor("input_257")]; - tensor input_259 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_257)[name = tensor("linear_45")]; - tensor input_261 = relu(x = input_259)[name = tensor("input_261")]; - tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_261)[name = tensor("linear_46")]; - tensor input_263 = add(x = input_257, y = ffn_out_1)[name = tensor("input_263")]; + tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; + tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1264, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; + tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; + tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; + tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1693, gamma = decoder_norm22_0_weight, x = input_263)[name = tensor("xt_3")]; - tensor var_1956 = const()[name = tensor("op_1956"), val = tensor([1, 3, 9, 256])]; - tensor x_47 = reshape(shape = var_1956, x = xt_3)[name = tensor("x_47")]; - tensor var_1958_perm_0 = const()[name = tensor("op_1958_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1962 = const()[name = tensor("op_1962"), val = tensor([9, 3, 256])]; - tensor var_1958 = transpose(perm = var_1958_perm_0, x = x_47)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_1962, x = var_1958)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1264, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1528 = const()[name = tensor("op_1528"), val = tensor([1, 3, 9, 256])]; + tensor x_31 = reshape(shape = var_1528, x = xt_3)[name = tensor("x_31")]; + tensor var_1530_perm_0 = const()[name = tensor("op_1530_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1534 = const()[name = tensor("op_1534"), val = tensor([9, 3, 256])]; + tensor var_1530 = transpose(perm = var_1530_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1534, x = var_1530)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 9, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1731,120 +1428,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_1970 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_1971 = const()[name = tensor("op_1971"), val = tensor([9, 3, 4, 64])]; - tensor var_1972 = reshape(shape = var_1971, x = var_1970)[name = tensor("op_1972")]; + tensor var_1542 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1543 = const()[name = tensor("op_1543"), val = tensor([9, 3, 4, 64])]; + tensor var_1544 = reshape(shape = var_1543, x = var_1542)[name = tensor("op_1544")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1976 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_1977 = const()[name = tensor("op_1977"), val = tensor(0x1p-3)]; - tensor var_1978 = mul(x = var_1976, y = var_1977)[name = tensor("op_1978")]; - tensor var_1979 = const()[name = tensor("op_1979"), val = tensor([9, 3, 4, 64])]; - tensor var_1980 = reshape(shape = var_1979, x = var_1978)[name = tensor("op_1980")]; + tensor var_1548 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1549 = const()[name = tensor("op_1549"), val = tensor(0x1p-3)]; + tensor var_1550 = mul(x = var_1548, y = var_1549)[name = tensor("op_1550")]; + tensor var_1551 = const()[name = tensor("op_1551"), val = tensor([9, 3, 4, 64])]; + tensor var_1552 = reshape(shape = var_1551, x = var_1550)[name = tensor("op_1552")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1984 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_1985 = const()[name = tensor("op_1985"), val = tensor([9, 3, 4, 64])]; - tensor var_1986 = reshape(shape = var_1985, x = var_1984)[name = tensor("op_1986")]; + tensor var_1556 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1557 = const()[name = tensor("op_1557"), val = tensor([9, 3, 4, 64])]; + tensor var_1558 = reshape(shape = var_1557, x = var_1556)[name = tensor("op_1558")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_267 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; + tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; - tensor const_40 = const()[name = tensor("const_40"), val = tensor(0x1.fffffep+127)]; - tensor clip_27 = clip(alpha = var_1698, beta = const_40, x = s_t)[name = tensor("clip_27")]; + tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; + tensor clip_27 = clip(alpha = var_1269, beta = const_32, x = s_t)[name = tensor("clip_27")]; tensor sqrt_s_t = sqrt(x = clip_27)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_1980)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_1972)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1552)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1544)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_2001 = const()[name = tensor("op_2001"), val = tensor([3, 1])]; - tensor var_2002 = reshape(shape = var_2001, x = sqrt_s_t)[name = tensor("op_2002")]; - tensor M = real_div(x = causal_with_valid_1, y = var_2002)[name = tensor("M")]; - tensor var_2004 = mul(x = qk, y = M)[name = tensor("op_2004")]; + tensor var_1573 = const()[name = tensor("op_1573"), val = tensor([3, 1])]; + tensor var_1574 = reshape(shape = var_1573, x = sqrt_s_t)[name = tensor("op_1574")]; + tensor M = real_div(x = causal_with_valid_1, y = var_1574)[name = tensor("M")]; + tensor var_1576 = mul(x = qk, y = M)[name = tensor("op_1576")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_1986)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_2004, y = v_17)[name = tensor("inner")]; - tensor var_2006_transpose_x_0 = const()[name = tensor("op_2006_transpose_x_0"), val = tensor(false)]; - tensor var_2006_transpose_y_0 = const()[name = tensor("op_2006_transpose_y_0"), val = tensor(false)]; - tensor var_2006 = matmul(transpose_x = var_2006_transpose_x_0, transpose_y = var_2006_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_2006")]; - tensor var_2007 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_2007")]; - tensor var_2008 = const()[name = tensor("op_2008"), val = tensor([1, 1, 3, 1])]; - tensor var_2009 = reshape(shape = var_2008, x = var_2007)[name = tensor("op_2009")]; - tensor cross = mul(x = var_2006, y = var_2009)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1558)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1576, y = v_17)[name = tensor("inner")]; + tensor var_1578_transpose_x_0 = const()[name = tensor("op_1578_transpose_x_0"), val = tensor(false)]; + tensor var_1578_transpose_y_0 = const()[name = tensor("op_1578_transpose_y_0"), val = tensor(false)]; + tensor var_1578 = matmul(transpose_x = var_1578_transpose_x_0, transpose_y = var_1578_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1578")]; + tensor var_1579 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1579")]; + tensor var_1580 = const()[name = tensor("op_1580"), val = tensor([1, 1, 3, 1])]; + tensor var_1581 = reshape(shape = var_1580, x = var_1579)[name = tensor("op_1581")]; + tensor cross = mul(x = var_1578, y = var_1581)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_1832)[name = tensor("v_masked")]; - tensor var_2015 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_2015")]; - tensor var_2017_transpose_x_1 = const()[name = tensor("op_2017_transpose_x_1"), val = tensor(true)]; - tensor var_2017_transpose_y_1 = const()[name = tensor("op_2017_transpose_y_1"), val = tensor(false)]; - tensor var_2017 = matmul(transpose_x = var_2017_transpose_x_1, transpose_y = var_2017_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_2017")]; - tensor new_kv_unnorm = add(x = var_2015, y = var_2017)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_1840)[name = tensor("new_scale")]; - tensor const_41 = const()[name = tensor("const_41"), val = tensor(0x1.fffffep+127)]; - tensor clip_28 = clip(alpha = var_1698, beta = const_41, x = new_scale)[name = tensor("clip_28")]; + tensor v_masked = mul(x = v_17, y = var_1404)[name = tensor("v_masked")]; + tensor var_1587 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1587")]; + tensor var_1589_transpose_x_1 = const()[name = tensor("op_1589_transpose_x_1"), val = tensor(true)]; + tensor var_1589_transpose_y_1 = const()[name = tensor("op_1589_transpose_y_1"), val = tensor(false)]; + tensor var_1589 = matmul(transpose_x = var_1589_transpose_x_1, transpose_y = var_1589_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1589")]; + tensor new_kv_unnorm = add(x = var_1587, y = var_1589)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1412)[name = tensor("new_scale")]; + tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; + tensor clip_28 = clip(alpha = var_1269, beta = const_33, x = new_scale)[name = tensor("clip_28")]; tensor sqrt_new_scale = sqrt(x = clip_28)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_2026_perm_0 = const()[name = tensor("op_2026_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1598_perm_0 = const()[name = tensor("op_1598_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_2026 = transpose(perm = var_2026_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1695, x = var_2026)[name = tensor("out_33")]; - tensor var_2030 = const()[name = tensor("op_2030"), val = tensor([9, 3, 256])]; - tensor out = reshape(shape = var_2030, x = out_33)[name = tensor("out")]; - tensor var_2032 = silu(x = input_267)[name = tensor("op_2032")]; - tensor input_269 = mul(x = var_2032, y = out)[name = tensor("input_269")]; - tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_269)[name = tensor("linear_51")]; - tensor input_271 = add(x = x, y = ret_out)[name = tensor("input_271")]; + tensor var_1598 = transpose(perm = var_1598_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1266, x = var_1598)[name = tensor("out_33")]; + tensor var_1602 = const()[name = tensor("op_1602"), val = tensor([9, 3, 256])]; + tensor out = reshape(shape = var_1602, x = out_33)[name = tensor("out")]; + tensor var_1604 = silu(x = input_187)[name = tensor("op_1604")]; + tensor input_189 = mul(x = var_1604, y = out)[name = tensor("input_189")]; + tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; + tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1693, gamma = decoder_norm11_1_weight, x = input_271)[name = tensor("xt_5")]; - tensor var_2042 = const()[name = tensor("op_2042"), val = tensor([1, 9, 3, 256])]; - tensor var_2043 = reshape(shape = var_2042, x = xt_5)[name = tensor("op_2043")]; - tensor var_2044_perm_0 = const()[name = tensor("op_2044_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2047 = const()[name = tensor("op_2047"), val = tensor([3, 9, 256])]; - tensor var_2044 = transpose(perm = var_2044_perm_0, x = var_2043)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_2047, x = var_2044)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1264, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1614 = const()[name = tensor("op_1614"), val = tensor([1, 9, 3, 256])]; + tensor var_1615 = reshape(shape = var_1614, x = xt_5)[name = tensor("op_1615")]; + tensor var_1616_perm_0 = const()[name = tensor("op_1616_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1619 = const()[name = tensor("op_1619"), val = tensor([3, 9, 256])]; + tensor var_1616 = transpose(perm = var_1616_perm_0, x = var_1615)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1619, x = var_1616)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_2070 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_13 = const()[name = tensor("concat_13"), val = tensor([9, 3, 3, 256])]; - tensor var_2072 = reshape(shape = concat_13, x = var_2070)[name = tensor("op_2072")]; - tensor var_2073_axes_0 = const()[name = tensor("op_2073_axes_0"), val = tensor([0])]; - tensor var_2073 = expand_dims(axes = var_2073_axes_0, x = var_2072)[name = tensor("op_2073")]; - tensor var_2074_perm_0 = const()[name = tensor("op_2074_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2075_axes_0 = const()[name = tensor("op_2075_axes_0"), val = tensor([-2])]; - tensor var_2074 = transpose(perm = var_2074_perm_0, x = var_2073)[name = tensor("transpose_8")]; - tensor var_2075 = squeeze(axes = var_2075_axes_0, x = var_2074)[name = tensor("op_2075")]; + tensor var_1642 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_14 = const()[name = tensor("concat_14"), val = tensor([9, 3, 3, 256])]; + tensor var_1644 = reshape(shape = concat_14, x = var_1642)[name = tensor("op_1644")]; + tensor var_1645_axes_0 = const()[name = tensor("op_1645_axes_0"), val = tensor([0])]; + tensor var_1645 = expand_dims(axes = var_1645_axes_0, x = var_1644)[name = tensor("op_1645")]; + tensor var_1646_perm_0 = const()[name = tensor("op_1646_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1647_axes_0 = const()[name = tensor("op_1647_axes_0"), val = tensor([-2])]; + tensor var_1646 = transpose(perm = var_1646_perm_0, x = var_1645)[name = tensor("transpose_8")]; + tensor var_1647 = squeeze(axes = var_1647_axes_0, x = var_1646)[name = tensor("op_1647")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 9, 3, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_2075)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1647)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 9, 3, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_2075)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1647)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 9, 3, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_2075)[name = tensor("v_19")]; - tensor var_2083 = const()[name = tensor("op_2083"), val = tensor([9, 12, 64])]; - tensor var_2084 = reshape(shape = var_2083, x = q_19)[name = tensor("op_2084")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1647)[name = tensor("v_19")]; + tensor var_1655 = const()[name = tensor("op_1655"), val = tensor([9, 12, 64])]; + tensor var_1656 = reshape(shape = var_1655, x = q_19)[name = tensor("op_1656")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2090 = const()[name = tensor("op_2090"), val = tensor([9, 12, 64])]; - tensor var_2091 = reshape(shape = var_2090, x = k_19)[name = tensor("op_2091")]; + tensor var_1662 = const()[name = tensor("op_1662"), val = tensor([9, 12, 64])]; + tensor var_1663 = reshape(shape = var_1662, x = k_19)[name = tensor("op_1663")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2097 = const()[name = tensor("op_2097"), val = tensor([9, 12, 64])]; - tensor var_2098 = reshape(shape = var_2097, x = v_19)[name = tensor("op_2098")]; + tensor var_1669 = const()[name = tensor("op_1669"), val = tensor([9, 12, 64])]; + tensor var_1670 = reshape(shape = var_1669, x = v_19)[name = tensor("op_1670")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2101 = const()[name = tensor("op_2101"), val = tensor([3, 4, 9, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_2084)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_2101, x = q_21)[name = tensor("q")]; - tensor var_2103 = const()[name = tensor("op_2103"), val = tensor([3, 4, 9, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_2091)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_2103, x = k_21)[name = tensor("k")]; - tensor var_2105 = const()[name = tensor("op_2105"), val = tensor([3, 4, 9, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_2098)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_2105, x = v_21)[name = tensor("v")]; + tensor var_1673 = const()[name = tensor("op_1673"), val = tensor([3, 4, 9, 64])]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1656)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1673, x = q_21)[name = tensor("q")]; + tensor var_1675 = const()[name = tensor("op_1675"), val = tensor([3, 4, 9, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1663)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1675, x = k_21)[name = tensor("k")]; + tensor var_1677 = const()[name = tensor("op_1677"), val = tensor([3, 4, 9, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1670)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1677, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -1855,50 +1552,50 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_2108 = const()[name = tensor("op_2108"), val = tensor([2, 0, 1, 3])]; - tensor var_2113 = const()[name = tensor("op_2113"), val = tensor([27, 256])]; - tensor var_2109 = transpose(perm = var_2108, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_2113, x = var_2109)[name = tensor("attn_output_11")]; + tensor var_1680 = const()[name = tensor("op_1680"), val = tensor([2, 0, 1, 3])]; + tensor var_1685 = const()[name = tensor("op_1685"), val = tensor([27, 256])]; + tensor var_1681 = transpose(perm = var_1680, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1685, x = var_1681)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_2117 = const()[name = tensor("op_2117"), val = tensor([9, 3, 256])]; - tensor attn_output = reshape(shape = var_2117, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1689 = const()[name = tensor("op_1689"), val = tensor([9, 3, 256])]; + tensor attn_output = reshape(shape = var_1689, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; - tensor input_273 = add(x = query_5, y = sa2_out)[name = tensor("input_273")]; - tensor input_275_axes_0 = const()[name = tensor("input_275_axes_0"), val = tensor([-1])]; - tensor input_275 = layer_norm(axes = input_275_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1693, gamma = decoder_norm21_1_weight, x = input_273)[name = tensor("input_275")]; - tensor input_277 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_275)[name = tensor("linear_54")]; - tensor input_279 = relu(x = input_277)[name = tensor("input_279")]; - tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_279)[name = tensor("linear_55")]; - tensor input_281 = add(x = input_275, y = ffn_out)[name = tensor("input_281")]; + tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; + tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1264, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; + tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; + tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; + tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1693, gamma = decoder_norm22_1_weight, x = input_281)[name = tensor("xt")]; - tensor var_2137 = const()[name = tensor("op_2137"), val = tensor([1, 3, 9, 256])]; - tensor input = reshape(shape = var_2137, x = xt)[name = tensor("input")]; - tensor var_2139 = const()[name = tensor("op_2139"), val = tensor([-1])]; - tensor var_2140 = reduce_l2_norm(axes = var_2139, keep_dims = var_1696, x = input)[name = tensor("op_2140")]; - tensor const_50 = const()[name = tensor("const_50"), val = tensor(0x1.fffffep+127)]; - tensor clip_29 = clip(alpha = var_1687, beta = const_50, x = var_2140)[name = tensor("clip_29")]; - tensor var_2142 = real_div(x = input, y = clip_29)[name = tensor("op_2142")]; - tensor concat_17 = const()[name = tensor("concat_17"), val = tensor([3, 1, 256])]; - tensor reshape_0 = reshape(shape = concat_17, x = emb)[name = tensor("reshape_0")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1264, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1709 = const()[name = tensor("op_1709"), val = tensor([1, 3, 9, 256])]; + tensor input = reshape(shape = var_1709, x = xt)[name = tensor("input")]; + tensor var_1711 = const()[name = tensor("op_1711"), val = tensor([-1])]; + tensor var_1712 = reduce_l2_norm(axes = var_1711, keep_dims = var_1267, x = input)[name = tensor("op_1712")]; + tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; + tensor clip_29 = clip(alpha = var_1259, beta = const_42, x = var_1712)[name = tensor("clip_29")]; + tensor var_1714 = real_div(x = input, y = clip_29)[name = tensor("op_1714")]; + tensor concat_18 = const()[name = tensor("concat_18"), val = tensor([3, 1, 256])]; + tensor reshape_0 = reshape(shape = concat_18, x = emb)[name = tensor("reshape_0")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_18 = const()[name = tensor("concat_18"), val = tensor([3, 256, 9])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_2142)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_18, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_19 = const()[name = tensor("concat_19"), val = tensor([3, 256, 9])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1714)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_19, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = reshape_0, y = reshape_1)[name = tensor("matmul_2")]; - tensor concat_21 = const()[name = tensor("concat_21"), val = tensor([1, 3, 9])]; - tensor reshape_2 = reshape(shape = concat_21, x = matmul_2)[name = tensor("reshape_2")]; + tensor concat_22 = const()[name = tensor("concat_22"), val = tensor([1, 3, 9])]; + tensor reshape_2 = reshape(shape = concat_22, x = matmul_2)[name = tensor("reshape_2")]; tensor output_begin_0 = const()[name = tensor("output_begin_0"), val = tensor([0, 0, 1])]; tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 3, 8])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = reshape_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_2146")]; - tensor var_2148_axis_0 = const()[name = tensor("op_2148_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_2148_axis_0, values = (var_1844, nkv))[name = tensor("op_2148")]; - tensor var_2150_axis_0 = const()[name = tensor("op_2150_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_2150_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_2150")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1718")]; + tensor var_1720_axis_0 = const()[name = tensor("op_1720_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1720_axis_0, values = (var_1416, nkv))[name = tensor("op_1720")]; + tensor var_1722_axis_0 = const()[name = tensor("op_1722_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1722_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1722")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/CALLHOME/ls_eend_ch_300ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/CALLHOME/ls_eend_ch_300ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index b7f061be9ffec4f87b5a379dbe6e3acb7df0a1fe..50174ea385b2da88325d486fec465beaa86180f8 100644 --- a/optimized/CALLHOME/ls_eend_ch_300ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/CALLHOME/ls_eend_ch_300ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f1f720acf2bee3d1a374ba0b6e6329efdc2078eed9d70dc10cfc514ee850d206 -size 269875 +oid sha256:feb79464f7e696aecf2fe909666b3bbdb61ec7b7d5731441bef6177625325327 +size 222326 diff --git a/optimized/CALLHOME/ls_eend_ch_300ms.mlpackage/Manifest.json b/optimized/CALLHOME/ls_eend_ch_300ms.mlpackage/Manifest.json index 811168b773634212a5ce315b10b53017122c9657..6dc6fa52792455fc526320044d95b0467263d4fa 100644 --- a/optimized/CALLHOME/ls_eend_ch_300ms.mlpackage/Manifest.json +++ b/optimized/CALLHOME/ls_eend_ch_300ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "678A92D0-3683-462C-AFCF-CEC10827365A": { - "author": "com.apple.CoreML", - "description": "CoreML Model Specification", - "name": "model.mlmodel", - "path": "com.apple.CoreML/model.mlmodel" - }, - "ECBB0F37-8123-429E-82E9-11181BB4D6EB": { + "592633F9-8850-455C-A11B-4EA9B2C6DB90": { "author": "com.apple.CoreML", "description": "CoreML Model Weights", "name": "weights", "path": "com.apple.CoreML/weights" + }, + "FDD9A5FA-02E0-4E84-9F0B-88FD33438585": { + "author": "com.apple.CoreML", + "description": "CoreML Model Specification", + "name": "model.mlmodel", + "path": "com.apple.CoreML/model.mlmodel" } }, - "rootModelIdentifier": "678A92D0-3683-462C-AFCF-CEC10827365A" + "rootModelIdentifier": "FDD9A5FA-02E0-4E84-9F0B-88FD33438585" } diff --git a/optimized/CALLHOME/ls_eend_ch_400ms.mlmodelc/analytics/coremldata.bin b/optimized/CALLHOME/ls_eend_ch_400ms.mlmodelc/analytics/coremldata.bin index f5d38f83b86673a9dac55ab8b12e3866a6a28a7e..36ea3265c34dee2bc6e436582dfd895fb6f25726 100644 --- a/optimized/CALLHOME/ls_eend_ch_400ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/CALLHOME/ls_eend_ch_400ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3c113bc48aed40ff59a357e2542c9ed823ee88c6fd981e042d7a7d418a9810dc +oid sha256:a2224d169efd3e336d04a23ca6aabe7443cdb705bd5bc122e63e1a2f67a96c39 size 243 diff --git a/optimized/CALLHOME/ls_eend_ch_400ms.mlmodelc/coremldata.bin b/optimized/CALLHOME/ls_eend_ch_400ms.mlmodelc/coremldata.bin index 3350d59dbe7b61091f7e4133794e3538479c3703..0c461c083ac9563d39cfd437a6e6b8d17eef6da3 100644 --- a/optimized/CALLHOME/ls_eend_ch_400ms.mlmodelc/coremldata.bin +++ b/optimized/CALLHOME/ls_eend_ch_400ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0fd57368bb16ac30fc8635c5bd68851f1115473fca06b650118ce1203edd8644 -size 1259 +oid sha256:ca1c1857b1a9db82d053d51086a934941cdfb6218c639273a79c309a3b6f78b7 +size 1350 diff --git a/optimized/CALLHOME/ls_eend_ch_400ms.mlmodelc/metadata.json b/optimized/CALLHOME/ls_eend_ch_400ms.mlmodelc/metadata.json index 8675bd4babd994cda49e501c1b10cbcb754877fa..65f6755875e9687372ad2ce0f61251b277ee3c55 100644 --- a/optimized/CALLHOME/ls_eend_ch_400ms.mlmodelc/metadata.json +++ b/optimized/CALLHOME/ls_eend_ch_400ms.mlmodelc/metadata.json @@ -96,30 +96,30 @@ "Ios17.reshape" : 100, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 77, - "Split" : 16, + "Ios17.transpose" : 57, + "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 94, "Tile" : 17, - "Ios17.sliceByIndex" : 92, - "Ios16.sigmoid" : 17, + "Ios17.sliceByIndex" : 68, + "Ios16.sigmoid" : 5, "Ios16.reduceSum" : 1, "Ios17.squeeze" : 2, - "Ios17.layerNorm" : 45, - "Ios17.batchNorm" : 16, + "Ios17.layerNorm" : 33, + "Ios17.batchNorm" : 4, "Ios17.cast" : 32, "Ios17.sqrt" : 18, "Ios17.sub" : 32, - "Ios17.conv" : 49, + "Ios17.conv" : 13, "Ios17.realDiv" : 20, "Ios17.linear" : 56, "Ios17.concat" : 22, "Ios17.clip" : 38, "Ios17.greaterEqual" : 16, - "Ios16.silu" : 30, + "Ios16.silu" : 18, "Stack" : 6, "Ios16.cumsum" : 1, - "Ios17.mul" : 123 + "Ios17.mul" : 111 }, "computePrecision" : "Mixed (Float32, Int32)", "isUpdatable" : "0", @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"ch\", \"model_label\": \"CALLHOME\", \"variant\": \"pipeline\", \"T\": 4, \"max_speakers\": 7, \"max_nspks\": 9, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"ch\", \"model_label\": \"CALLHOME\", \"variant\": \"pipeline\", \"chunk_size\": 4, \"step_duration_ms\": 400, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 7, \"max_nspks\": 9, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/CALLHOME/ls_eend_ch_400ms.mlmodelc/model.mil b/optimized/CALLHOME/ls_eend_ch_400ms.mlmodelc/model.mil index 551ffa450f27e4c7ec64142208e61e0f905caa35..bbd4b62b28583aea8e0141c4ba066d938f1ca9aa 100644 --- a/optimized/CALLHOME/ls_eend_ch_400ms.mlmodelc/model.mil +++ b/optimized/CALLHOME/ls_eend_ch_400ms.mlmodelc/model.mil @@ -217,6 +217,7 @@ program(1.0) tensor var_21 = const()[name = tensor("op_21"), val = tensor(0x0p+0)]; tensor var_24 = const()[name = tensor("op_24"), val = tensor(0x1.0c6f7ap-20)]; tensor var_27 = const()[name = tensor("op_27"), val = tensor(2)]; + tensor var_30 = const()[name = tensor("op_30"), val = tensor(0)]; tensor var_33 = const()[name = tensor("op_33"), val = tensor(1)]; tensor var_35 = const()[name = tensor("op_35"), val = tensor(0x1.4f8b58p-17)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(true)]; @@ -263,8 +264,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_77")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_78")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_196 = const()[name = tensor("op_196"), val = tensor([4, 1])]; tensor var_197 = reshape(shape = var_196, x = sqrt_s_t_1)[name = tensor("op_197")]; @@ -272,7 +273,7 @@ program(1.0) tensor var_199 = mul(x = qk_1, y = M_1)[name = tensor("op_199")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_76")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_199, y = v_1)[name = tensor("inner_1")]; tensor var_201_transpose_x_0 = const()[name = tensor("op_201_transpose_x_0"), val = tensor(false)]; tensor var_201_transpose_y_0 = const()[name = tensor("op_201_transpose_y_0"), val = tensor(false)]; @@ -293,7 +294,7 @@ program(1.0) tensor var_214 = real_div(x = new_kv_unnorm_1, y = var_213)[name = tensor("op_214")]; tensor var_215_perm_0 = const()[name = tensor("op_215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_75")]; + tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_24, x = var_215)[name = tensor("out_3")]; tensor var_219 = const()[name = tensor("op_219"), val = tensor([1, 4, 256])]; tensor out_5 = reshape(shape = var_219, x = out_3)[name = tensor("out_5")]; @@ -341,291 +342,176 @@ program(1.0) tensor var_252 = sub(x = var_17, y = use_roll_1)[name = tensor("op_252")]; tensor var_253 = mul(x = filled_window_1, y = var_252)[name = tensor("op_253")]; tensor var_254 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_254")]; - tensor input_21 = add(x = var_253, y = var_254)[name = tensor("input_21")]; + tensor window_3 = add(x = var_253, y = var_254)[name = tensor("window_3")]; tensor var_256 = const()[name = tensor("op_256"), val = tensor(0x1p+0)]; tensor var_257 = add(x = filled_1, y = var_256)[name = tensor("op_257")]; tensor clip_1 = clip(alpha = var_21, beta = var_16, x = var_257)[name = tensor("clip_1")]; + tensor var_260_begin_0 = const()[name = tensor("op_260_begin_0"), val = tensor([0, 1, 0])]; + tensor var_260_end_0 = const()[name = tensor("op_260_end_0"), val = tensor([1, 2, 256])]; + tensor var_260_end_mask_0 = const()[name = tensor("op_260_end_mask_0"), val = tensor([true, false, true])]; + tensor var_260 = slice_by_index(begin = var_260_begin_0, end = var_260_end_0, end_mask = var_260_end_mask_0, x = x_3)[name = tensor("op_260")]; + tensor var_263_begin_0 = const()[name = tensor("op_263_begin_0"), val = tensor([0, 1, 0])]; + tensor var_263_end_0 = const()[name = tensor("op_263_end_0"), val = tensor([1, 16, 256])]; + tensor var_263_end_mask_0 = const()[name = tensor("op_263_end_mask_0"), val = tensor([true, true, true])]; + tensor var_263 = slice_by_index(begin = var_263_begin_0, end = var_263_end_0, end_mask = var_263_end_mask_0, x = window_3)[name = tensor("op_263")]; + tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; + tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_263, var_260))[name = tensor("rolled_3")]; + tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; + tensor var_268 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_268")]; + tensor var_269_dtype_0 = const()[name = tensor("op_269_dtype_0"), val = tensor("fp32")]; + tensor var_270 = const()[name = tensor("op_270"), val = tensor([1, 16, 1])]; + tensor var_269 = cast(dtype = var_269_dtype_0, x = var_268)[name = tensor("cast_173")]; + tensor fill_mask_3 = reshape(shape = var_270, x = var_269)[name = tensor("fill_mask_3")]; + tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_260)[name = tensor("x_expanded_3")]; + tensor var_274 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_274")]; + tensor var_275 = mul(x = window_3, y = var_274)[name = tensor("op_275")]; + tensor var_276 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_276")]; + tensor filled_window_3 = add(x = var_275, y = var_276)[name = tensor("filled_window_3")]; + tensor var_278 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_278")]; + tensor var_279_dtype_0 = const()[name = tensor("op_279_dtype_0"), val = tensor("fp32")]; + tensor var_280 = const()[name = tensor("op_280"), val = tensor([1, 1, 1])]; + tensor var_279 = cast(dtype = var_279_dtype_0, x = var_278)[name = tensor("cast_172")]; + tensor use_roll_3 = reshape(shape = var_280, x = var_279)[name = tensor("use_roll_3")]; + tensor var_282 = sub(x = var_17, y = use_roll_3)[name = tensor("op_282")]; + tensor var_283 = mul(x = filled_window_3, y = var_282)[name = tensor("op_283")]; + tensor var_284 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_284")]; + tensor window_5 = add(x = var_283, y = var_284)[name = tensor("window_5")]; + tensor var_286 = const()[name = tensor("op_286"), val = tensor(0x1p+0)]; + tensor var_287 = add(x = clip_1, y = var_286)[name = tensor("op_287")]; + tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_287)[name = tensor("clip_3")]; + tensor var_290_begin_0 = const()[name = tensor("op_290_begin_0"), val = tensor([0, 2, 0])]; + tensor var_290_end_0 = const()[name = tensor("op_290_end_0"), val = tensor([1, 3, 256])]; + tensor var_290_end_mask_0 = const()[name = tensor("op_290_end_mask_0"), val = tensor([true, false, true])]; + tensor var_290 = slice_by_index(begin = var_290_begin_0, end = var_290_end_0, end_mask = var_290_end_mask_0, x = x_3)[name = tensor("op_290")]; + tensor var_293_begin_0 = const()[name = tensor("op_293_begin_0"), val = tensor([0, 1, 0])]; + tensor var_293_end_0 = const()[name = tensor("op_293_end_0"), val = tensor([1, 16, 256])]; + tensor var_293_end_mask_0 = const()[name = tensor("op_293_end_mask_0"), val = tensor([true, true, true])]; + tensor var_293 = slice_by_index(begin = var_293_begin_0, end = var_293_end_0, end_mask = var_293_end_mask_0, x = window_5)[name = tensor("op_293")]; + tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; + tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_293, var_290))[name = tensor("rolled_5")]; + tensor clip_4 = clip(alpha = var_21, beta = var_20, x = clip_3)[name = tensor("clip_4")]; + tensor var_298 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_298")]; + tensor var_299_dtype_0 = const()[name = tensor("op_299_dtype_0"), val = tensor("fp32")]; + tensor var_300 = const()[name = tensor("op_300"), val = tensor([1, 16, 1])]; + tensor var_299 = cast(dtype = var_299_dtype_0, x = var_298)[name = tensor("cast_171")]; + tensor fill_mask_5 = reshape(shape = var_300, x = var_299)[name = tensor("fill_mask_5")]; + tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_290)[name = tensor("x_expanded_5")]; + tensor var_304 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_304")]; + tensor var_305 = mul(x = window_5, y = var_304)[name = tensor("op_305")]; + tensor var_306 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_306")]; + tensor filled_window_5 = add(x = var_305, y = var_306)[name = tensor("filled_window_5")]; + tensor var_308 = greater_equal(x = clip_3, y = var_16)[name = tensor("op_308")]; + tensor var_309_dtype_0 = const()[name = tensor("op_309_dtype_0"), val = tensor("fp32")]; + tensor var_310 = const()[name = tensor("op_310"), val = tensor([1, 1, 1])]; + tensor var_309 = cast(dtype = var_309_dtype_0, x = var_308)[name = tensor("cast_170")]; + tensor use_roll_5 = reshape(shape = var_310, x = var_309)[name = tensor("use_roll_5")]; + tensor var_312 = sub(x = var_17, y = use_roll_5)[name = tensor("op_312")]; + tensor var_313 = mul(x = filled_window_5, y = var_312)[name = tensor("op_313")]; + tensor var_314 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_314")]; + tensor window_7 = add(x = var_313, y = var_314)[name = tensor("window_7")]; + tensor var_316 = const()[name = tensor("op_316"), val = tensor(0x1p+0)]; + tensor var_317 = add(x = clip_3, y = var_316)[name = tensor("op_317")]; + tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_317)[name = tensor("clip_5")]; + tensor var_320_begin_0 = const()[name = tensor("op_320_begin_0"), val = tensor([0, 3, 0])]; + tensor var_320_end_0 = const()[name = tensor("op_320_end_0"), val = tensor([1, 1, 256])]; + tensor var_320_end_mask_0 = const()[name = tensor("op_320_end_mask_0"), val = tensor([true, true, true])]; + tensor var_320 = slice_by_index(begin = var_320_begin_0, end = var_320_end_0, end_mask = var_320_end_mask_0, x = x_3)[name = tensor("op_320")]; + tensor var_323_begin_0 = const()[name = tensor("op_323_begin_0"), val = tensor([0, 1, 0])]; + tensor var_323_end_0 = const()[name = tensor("op_323_end_0"), val = tensor([1, 16, 256])]; + tensor var_323_end_mask_0 = const()[name = tensor("op_323_end_mask_0"), val = tensor([true, true, true])]; + tensor var_323 = slice_by_index(begin = var_323_begin_0, end = var_323_end_0, end_mask = var_323_end_mask_0, x = window_7)[name = tensor("op_323")]; + tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; + tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_323, var_320))[name = tensor("rolled_7")]; + tensor clip_6 = clip(alpha = var_21, beta = var_20, x = clip_5)[name = tensor("clip_6")]; + tensor var_328 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_328")]; + tensor var_329_dtype_0 = const()[name = tensor("op_329_dtype_0"), val = tensor("fp32")]; + tensor var_330 = const()[name = tensor("op_330"), val = tensor([1, 16, 1])]; + tensor var_329 = cast(dtype = var_329_dtype_0, x = var_328)[name = tensor("cast_169")]; + tensor fill_mask_7 = reshape(shape = var_330, x = var_329)[name = tensor("fill_mask_7")]; + tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_320)[name = tensor("x_expanded_7")]; + tensor var_334 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_334")]; + tensor var_335 = mul(x = window_7, y = var_334)[name = tensor("op_335")]; + tensor var_336 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_336")]; + tensor filled_window_7 = add(x = var_335, y = var_336)[name = tensor("filled_window_7")]; + tensor var_338 = greater_equal(x = clip_5, y = var_16)[name = tensor("op_338")]; + tensor var_339_dtype_0 = const()[name = tensor("op_339_dtype_0"), val = tensor("fp32")]; + tensor var_340 = const()[name = tensor("op_340"), val = tensor([1, 1, 1])]; + tensor var_339 = cast(dtype = var_339_dtype_0, x = var_338)[name = tensor("cast_168")]; + tensor use_roll_7 = reshape(shape = var_340, x = var_339)[name = tensor("use_roll_7")]; + tensor var_342 = sub(x = var_17, y = use_roll_7)[name = tensor("op_342")]; + tensor var_343 = mul(x = filled_window_7, y = var_342)[name = tensor("op_343")]; + tensor var_344 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_344")]; + tensor window_9 = add(x = var_343, y = var_344)[name = tensor("window_9")]; + tensor var_346 = const()[name = tensor("op_346"), val = tensor(0x1p+0)]; + tensor var_347 = add(x = clip_5, y = var_346)[name = tensor("op_347")]; + tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_347)[name = tensor("clip_7")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_30, interleave = input_21_interleave_0, values = (window_3, window_5, window_7, window_9))[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; - tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; + tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; tensor inputs_3_pad_type_0 = const()[name = tensor("inputs_3_pad_type_0"), val = tensor("valid")]; tensor inputs_3_strides_0 = const()[name = tensor("inputs_3_strides_0"), val = tensor([1])]; tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_74")]; - tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_278_split_sizes_0 = const()[name = tensor("op_278_split_sizes_0"), val = tensor([256, 256])]; - tensor var_278_axis_0 = const()[name = tensor("op_278_axis_0"), val = tensor(1)]; - tensor var_278_0, tensor var_278_1 = split(axis = var_278_axis_0, split_sizes = var_278_split_sizes_0, x = inputs_3)[name = tensor("op_278")]; - tensor var_280 = sigmoid(x = var_278_1)[name = tensor("op_280")]; - tensor inputs_5 = mul(x = var_278_0, y = var_280)[name = tensor("inputs_5")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; + tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; + tensor var_370_split_sizes_0 = const()[name = tensor("op_370_split_sizes_0"), val = tensor([256, 256])]; + tensor var_370_axis_0 = const()[name = tensor("op_370_axis_0"), val = tensor(1)]; + tensor var_370_0, tensor var_370_1 = split(axis = var_370_axis_0, split_sizes = var_370_split_sizes_0, x = inputs_3)[name = tensor("op_370")]; + tensor var_372 = sigmoid(x = var_370_1)[name = tensor("op_372")]; + tensor inputs_5 = mul(x = var_370_0, y = var_372)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; tensor outputs_aug_1_strides_0 = const()[name = tensor("outputs_aug_1_strides_0"), val = tensor([1])]; tensor outputs_aug_1_dilations_0 = const()[name = tensor("outputs_aug_1_dilations_0"), val = tensor([1])]; - tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; + tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; tensor input_25_begin_0 = const()[name = tensor("input_25_begin_0"), val = tensor([0, 0, 0])]; - tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([1, 256, 16])]; + tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([4, 256, 16])]; tensor input_25_end_mask_0 = const()[name = tensor("input_25_end_mask_0"), val = tensor([true, true, false])]; - tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; - tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; - tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; + tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; + tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; + tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; tensor input_29_pad_type_0 = const()[name = tensor("input_29_pad_type_0"), val = tensor("valid")]; tensor input_29_strides_0 = const()[name = tensor("input_29_strides_0"), val = tensor([1])]; tensor input_29_pad_0 = const()[name = tensor("input_29_pad_0"), val = tensor([0, 0])]; tensor input_29_dilations_0 = const()[name = tensor("input_29_dilations_0"), val = tensor([1])]; tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; - tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; + tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_311_begin_0 = const()[name = tensor("op_311_begin_0"), val = tensor([0, -1, 0])]; - tensor var_311_end_0 = const()[name = tensor("op_311_end_0"), val = tensor([1, 16, 256])]; - tensor var_311_end_mask_0 = const()[name = tensor("op_311_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_73")]; - tensor var_311 = slice_by_index(begin = var_311_begin_0, end = var_311_end_0, end_mask = var_311_end_mask_0, x = conv_out_1)[name = tensor("op_311")]; - tensor var_314_begin_0 = const()[name = tensor("op_314_begin_0"), val = tensor([0, 1, 0])]; - tensor var_314_end_0 = const()[name = tensor("op_314_end_0"), val = tensor([1, 2, 256])]; - tensor var_314_end_mask_0 = const()[name = tensor("op_314_end_mask_0"), val = tensor([true, false, true])]; - tensor var_314 = slice_by_index(begin = var_314_begin_0, end = var_314_end_0, end_mask = var_314_end_mask_0, x = x_3)[name = tensor("op_314")]; - tensor var_317_begin_0 = const()[name = tensor("op_317_begin_0"), val = tensor([0, 1, 0])]; - tensor var_317_end_0 = const()[name = tensor("op_317_end_0"), val = tensor([1, 16, 256])]; - tensor var_317_end_mask_0 = const()[name = tensor("op_317_end_mask_0"), val = tensor([true, true, true])]; - tensor var_317 = slice_by_index(begin = var_317_begin_0, end = var_317_end_0, end_mask = var_317_end_mask_0, x = input_21)[name = tensor("op_317")]; - tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_317, var_314))[name = tensor("rolled_3")]; - tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; - tensor var_322 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_322")]; - tensor var_323_dtype_0 = const()[name = tensor("op_323_dtype_0"), val = tensor("fp32")]; - tensor var_324 = const()[name = tensor("op_324"), val = tensor([1, 16, 1])]; - tensor var_323 = cast(dtype = var_323_dtype_0, x = var_322)[name = tensor("cast_173")]; - tensor fill_mask_3 = reshape(shape = var_324, x = var_323)[name = tensor("fill_mask_3")]; - tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_314)[name = tensor("x_expanded_3")]; - tensor var_328 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_328")]; - tensor var_329 = mul(x = input_21, y = var_328)[name = tensor("op_329")]; - tensor var_330 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_330")]; - tensor filled_window_3 = add(x = var_329, y = var_330)[name = tensor("filled_window_3")]; - tensor var_332 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_332")]; - tensor var_333_dtype_0 = const()[name = tensor("op_333_dtype_0"), val = tensor("fp32")]; - tensor var_334 = const()[name = tensor("op_334"), val = tensor([1, 1, 1])]; - tensor var_333 = cast(dtype = var_333_dtype_0, x = var_332)[name = tensor("cast_172")]; - tensor use_roll_3 = reshape(shape = var_334, x = var_333)[name = tensor("use_roll_3")]; - tensor var_336 = sub(x = var_17, y = use_roll_3)[name = tensor("op_336")]; - tensor var_337 = mul(x = filled_window_3, y = var_336)[name = tensor("op_337")]; - tensor var_338 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_338")]; - tensor input_31 = add(x = var_337, y = var_338)[name = tensor("input_31")]; - tensor var_340 = const()[name = tensor("op_340"), val = tensor(0x1p+0)]; - tensor var_341 = add(x = clip_1, y = var_340)[name = tensor("op_341")]; - tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_341)[name = tensor("clip_3")]; + tensor var_403_begin_0 = const()[name = tensor("op_403_begin_0"), val = tensor([0, -1, 0])]; + tensor var_403_end_0 = const()[name = tensor("op_403_end_0"), val = tensor([4, 16, 256])]; + tensor var_403_end_mask_0 = const()[name = tensor("op_403_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_403 = slice_by_index(begin = var_403_begin_0, end = var_403_end_0, end_mask = var_403_end_mask_0, x = conv_out_1)[name = tensor("op_403")]; + tensor var_405_perm_0 = const()[name = tensor("op_405_perm_0"), val = tensor([1, 0, 2])]; + tensor var_405 = transpose(perm = var_405_perm_0, x = var_403)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_405)[name = tensor("input_31")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; + tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; + tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; + tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; + tensor var_428 = const()[name = tensor("op_428"), val = tensor(0x1p-1)]; + tensor var_429 = mul(x = input_39, y = var_428)[name = tensor("op_429")]; + tensor input_41 = add(x = var_429, y = input_31)[name = tensor("input_41")]; + tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; + tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; + tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; + tensor input_45 = layer_norm(axes = input_45_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_43)[name = tensor("input_45")]; + tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; + tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; + tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; + tensor var_458 = const()[name = tensor("op_458"), val = tensor(0x1p-1)]; + tensor var_459 = mul(x = input_51, y = var_458)[name = tensor("op_459")]; + tensor input_53 = add(x = var_459, y = input_43)[name = tensor("input_53")]; tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; - tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_31)[name = tensor("x_7")]; - tensor input_33_perm_0 = const()[name = tensor("input_33_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_9_pad_type_0 = const()[name = tensor("inputs_9_pad_type_0"), val = tensor("valid")]; - tensor inputs_9_strides_0 = const()[name = tensor("inputs_9_strides_0"), val = tensor([1])]; - tensor inputs_9_pad_0 = const()[name = tensor("inputs_9_pad_0"), val = tensor([0, 0])]; - tensor inputs_9_dilations_0 = const()[name = tensor("inputs_9_dilations_0"), val = tensor([1])]; - tensor inputs_9_groups_0 = const()[name = tensor("inputs_9_groups_0"), val = tensor(1)]; - tensor input_33 = transpose(perm = input_33_perm_0, x = x_7)[name = tensor("transpose_72")]; - tensor inputs_9 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_9_dilations_0, groups = inputs_9_groups_0, pad = inputs_9_pad_0, pad_type = inputs_9_pad_type_0, strides = inputs_9_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_33)[name = tensor("inputs_9")]; - tensor var_362_split_sizes_0 = const()[name = tensor("op_362_split_sizes_0"), val = tensor([256, 256])]; - tensor var_362_axis_0 = const()[name = tensor("op_362_axis_0"), val = tensor(1)]; - tensor var_362_0, tensor var_362_1 = split(axis = var_362_axis_0, split_sizes = var_362_split_sizes_0, x = inputs_9)[name = tensor("op_362")]; - tensor var_364 = sigmoid(x = var_362_1)[name = tensor("op_364")]; - tensor inputs_11 = mul(x = var_362_0, y = var_364)[name = tensor("inputs_11")]; - tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; - tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; - tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; - tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_11)[name = tensor("outputs_aug_3")]; - tensor input_35_begin_0 = const()[name = tensor("input_35_begin_0"), val = tensor([0, 0, 0])]; - tensor input_35_end_0 = const()[name = tensor("input_35_end_0"), val = tensor([1, 256, 16])]; - tensor input_35_end_mask_0 = const()[name = tensor("input_35_end_mask_0"), val = tensor([true, true, false])]; - tensor input_35 = slice_by_index(begin = input_35_begin_0, end = input_35_end_0, end_mask = input_35_end_mask_0, x = outputs_aug_3)[name = tensor("input_35")]; - tensor inputs_13 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_35)[name = tensor("inputs_13")]; - tensor input_37 = silu(x = inputs_13)[name = tensor("input_37")]; - tensor input_39_pad_type_0 = const()[name = tensor("input_39_pad_type_0"), val = tensor("valid")]; - tensor input_39_strides_0 = const()[name = tensor("input_39_strides_0"), val = tensor([1])]; - tensor input_39_pad_0 = const()[name = tensor("input_39_pad_0"), val = tensor([0, 0])]; - tensor input_39_dilations_0 = const()[name = tensor("input_39_dilations_0"), val = tensor([1])]; - tensor input_39_groups_0 = const()[name = tensor("input_39_groups_0"), val = tensor(1)]; - tensor input_39 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_37)[name = tensor("input_39")]; - tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_395_begin_0 = const()[name = tensor("op_395_begin_0"), val = tensor([0, -1, 0])]; - tensor var_395_end_0 = const()[name = tensor("op_395_end_0"), val = tensor([1, 16, 256])]; - tensor var_395_end_mask_0 = const()[name = tensor("op_395_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_39)[name = tensor("transpose_71")]; - tensor var_395 = slice_by_index(begin = var_395_begin_0, end = var_395_end_0, end_mask = var_395_end_mask_0, x = conv_out_3)[name = tensor("op_395")]; - tensor var_398_begin_0 = const()[name = tensor("op_398_begin_0"), val = tensor([0, 2, 0])]; - tensor var_398_end_0 = const()[name = tensor("op_398_end_0"), val = tensor([1, 3, 256])]; - tensor var_398_end_mask_0 = const()[name = tensor("op_398_end_mask_0"), val = tensor([true, false, true])]; - tensor var_398 = slice_by_index(begin = var_398_begin_0, end = var_398_end_0, end_mask = var_398_end_mask_0, x = x_3)[name = tensor("op_398")]; - tensor var_401_begin_0 = const()[name = tensor("op_401_begin_0"), val = tensor([0, 1, 0])]; - tensor var_401_end_0 = const()[name = tensor("op_401_end_0"), val = tensor([1, 16, 256])]; - tensor var_401_end_mask_0 = const()[name = tensor("op_401_end_mask_0"), val = tensor([true, true, true])]; - tensor var_401 = slice_by_index(begin = var_401_begin_0, end = var_401_end_0, end_mask = var_401_end_mask_0, x = input_31)[name = tensor("op_401")]; - tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_401, var_398))[name = tensor("rolled_5")]; - tensor clip_4 = clip(alpha = var_21, beta = var_20, x = clip_3)[name = tensor("clip_4")]; - tensor var_406 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_406")]; - tensor var_407_dtype_0 = const()[name = tensor("op_407_dtype_0"), val = tensor("fp32")]; - tensor var_408 = const()[name = tensor("op_408"), val = tensor([1, 16, 1])]; - tensor var_407 = cast(dtype = var_407_dtype_0, x = var_406)[name = tensor("cast_171")]; - tensor fill_mask_5 = reshape(shape = var_408, x = var_407)[name = tensor("fill_mask_5")]; - tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_398)[name = tensor("x_expanded_5")]; - tensor var_412 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_412")]; - tensor var_413 = mul(x = input_31, y = var_412)[name = tensor("op_413")]; - tensor var_414 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_414")]; - tensor filled_window_5 = add(x = var_413, y = var_414)[name = tensor("filled_window_5")]; - tensor var_416 = greater_equal(x = clip_3, y = var_16)[name = tensor("op_416")]; - tensor var_417_dtype_0 = const()[name = tensor("op_417_dtype_0"), val = tensor("fp32")]; - tensor var_418 = const()[name = tensor("op_418"), val = tensor([1, 1, 1])]; - tensor var_417 = cast(dtype = var_417_dtype_0, x = var_416)[name = tensor("cast_170")]; - tensor use_roll_5 = reshape(shape = var_418, x = var_417)[name = tensor("use_roll_5")]; - tensor var_420 = sub(x = var_17, y = use_roll_5)[name = tensor("op_420")]; - tensor var_421 = mul(x = filled_window_5, y = var_420)[name = tensor("op_421")]; - tensor var_422 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_422")]; - tensor input_41 = add(x = var_421, y = var_422)[name = tensor("input_41")]; - tensor var_424 = const()[name = tensor("op_424"), val = tensor(0x1p+0)]; - tensor var_425 = add(x = clip_3, y = var_424)[name = tensor("op_425")]; - tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_425)[name = tensor("clip_5")]; - tensor x_9_axes_0 = const()[name = tensor("x_9_axes_0"), val = tensor([-1])]; - tensor x_9 = layer_norm(axes = x_9_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_41)[name = tensor("x_9")]; - tensor input_43_perm_0 = const()[name = tensor("input_43_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_15_pad_type_0 = const()[name = tensor("inputs_15_pad_type_0"), val = tensor("valid")]; - tensor inputs_15_strides_0 = const()[name = tensor("inputs_15_strides_0"), val = tensor([1])]; - tensor inputs_15_pad_0 = const()[name = tensor("inputs_15_pad_0"), val = tensor([0, 0])]; - tensor inputs_15_dilations_0 = const()[name = tensor("inputs_15_dilations_0"), val = tensor([1])]; - tensor inputs_15_groups_0 = const()[name = tensor("inputs_15_groups_0"), val = tensor(1)]; - tensor input_43 = transpose(perm = input_43_perm_0, x = x_9)[name = tensor("transpose_70")]; - tensor inputs_15 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_15_dilations_0, groups = inputs_15_groups_0, pad = inputs_15_pad_0, pad_type = inputs_15_pad_type_0, strides = inputs_15_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_43)[name = tensor("inputs_15")]; - tensor var_446_split_sizes_0 = const()[name = tensor("op_446_split_sizes_0"), val = tensor([256, 256])]; - tensor var_446_axis_0 = const()[name = tensor("op_446_axis_0"), val = tensor(1)]; - tensor var_446_0, tensor var_446_1 = split(axis = var_446_axis_0, split_sizes = var_446_split_sizes_0, x = inputs_15)[name = tensor("op_446")]; - tensor var_448 = sigmoid(x = var_446_1)[name = tensor("op_448")]; - tensor inputs_17 = mul(x = var_446_0, y = var_448)[name = tensor("inputs_17")]; - tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; - tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; - tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; - tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_17)[name = tensor("outputs_aug_5")]; - tensor input_45_begin_0 = const()[name = tensor("input_45_begin_0"), val = tensor([0, 0, 0])]; - tensor input_45_end_0 = const()[name = tensor("input_45_end_0"), val = tensor([1, 256, 16])]; - tensor input_45_end_mask_0 = const()[name = tensor("input_45_end_mask_0"), val = tensor([true, true, false])]; - tensor input_45 = slice_by_index(begin = input_45_begin_0, end = input_45_end_0, end_mask = input_45_end_mask_0, x = outputs_aug_5)[name = tensor("input_45")]; - tensor inputs_19 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_45)[name = tensor("inputs_19")]; - tensor input_47 = silu(x = inputs_19)[name = tensor("input_47")]; - tensor input_49_pad_type_0 = const()[name = tensor("input_49_pad_type_0"), val = tensor("valid")]; - tensor input_49_strides_0 = const()[name = tensor("input_49_strides_0"), val = tensor([1])]; - tensor input_49_pad_0 = const()[name = tensor("input_49_pad_0"), val = tensor([0, 0])]; - tensor input_49_dilations_0 = const()[name = tensor("input_49_dilations_0"), val = tensor([1])]; - tensor input_49_groups_0 = const()[name = tensor("input_49_groups_0"), val = tensor(1)]; - tensor input_49 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_49_dilations_0, groups = input_49_groups_0, pad = input_49_pad_0, pad_type = input_49_pad_type_0, strides = input_49_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_47)[name = tensor("input_49")]; - tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_479_begin_0 = const()[name = tensor("op_479_begin_0"), val = tensor([0, -1, 0])]; - tensor var_479_end_0 = const()[name = tensor("op_479_end_0"), val = tensor([1, 16, 256])]; - tensor var_479_end_mask_0 = const()[name = tensor("op_479_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_49)[name = tensor("transpose_69")]; - tensor var_479 = slice_by_index(begin = var_479_begin_0, end = var_479_end_0, end_mask = var_479_end_mask_0, x = conv_out_5)[name = tensor("op_479")]; - tensor var_482_begin_0 = const()[name = tensor("op_482_begin_0"), val = tensor([0, 3, 0])]; - tensor var_482_end_0 = const()[name = tensor("op_482_end_0"), val = tensor([1, 1, 256])]; - tensor var_482_end_mask_0 = const()[name = tensor("op_482_end_mask_0"), val = tensor([true, true, true])]; - tensor var_482 = slice_by_index(begin = var_482_begin_0, end = var_482_end_0, end_mask = var_482_end_mask_0, x = x_3)[name = tensor("op_482")]; - tensor var_485_begin_0 = const()[name = tensor("op_485_begin_0"), val = tensor([0, 1, 0])]; - tensor var_485_end_0 = const()[name = tensor("op_485_end_0"), val = tensor([1, 16, 256])]; - tensor var_485_end_mask_0 = const()[name = tensor("op_485_end_mask_0"), val = tensor([true, true, true])]; - tensor var_485 = slice_by_index(begin = var_485_begin_0, end = var_485_end_0, end_mask = var_485_end_mask_0, x = input_41)[name = tensor("op_485")]; - tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; - tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_485, var_482))[name = tensor("rolled_7")]; - tensor clip_6 = clip(alpha = var_21, beta = var_20, x = clip_5)[name = tensor("clip_6")]; - tensor var_490 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_490")]; - tensor var_491_dtype_0 = const()[name = tensor("op_491_dtype_0"), val = tensor("fp32")]; - tensor var_492 = const()[name = tensor("op_492"), val = tensor([1, 16, 1])]; - tensor var_491 = cast(dtype = var_491_dtype_0, x = var_490)[name = tensor("cast_169")]; - tensor fill_mask_7 = reshape(shape = var_492, x = var_491)[name = tensor("fill_mask_7")]; - tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_482)[name = tensor("x_expanded_7")]; - tensor var_496 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_496")]; - tensor var_497 = mul(x = input_41, y = var_496)[name = tensor("op_497")]; - tensor var_498 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_498")]; - tensor filled_window_7 = add(x = var_497, y = var_498)[name = tensor("filled_window_7")]; - tensor var_500 = greater_equal(x = clip_5, y = var_16)[name = tensor("op_500")]; - tensor var_501_dtype_0 = const()[name = tensor("op_501_dtype_0"), val = tensor("fp32")]; - tensor var_502 = const()[name = tensor("op_502"), val = tensor([1, 1, 1])]; - tensor var_501 = cast(dtype = var_501_dtype_0, x = var_500)[name = tensor("cast_168")]; - tensor use_roll_7 = reshape(shape = var_502, x = var_501)[name = tensor("use_roll_7")]; - tensor var_504 = sub(x = var_17, y = use_roll_7)[name = tensor("op_504")]; - tensor var_505 = mul(x = filled_window_7, y = var_504)[name = tensor("op_505")]; - tensor var_506 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_506")]; - tensor input_51 = add(x = var_505, y = var_506)[name = tensor("input_51")]; - tensor var_508 = const()[name = tensor("op_508"), val = tensor(0x1p+0)]; - tensor var_509 = add(x = clip_5, y = var_508)[name = tensor("op_509")]; - tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_509)[name = tensor("clip_7")]; - tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; - tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_51)[name = tensor("x_11")]; - tensor input_53_perm_0 = const()[name = tensor("input_53_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_21_pad_type_0 = const()[name = tensor("inputs_21_pad_type_0"), val = tensor("valid")]; - tensor inputs_21_strides_0 = const()[name = tensor("inputs_21_strides_0"), val = tensor([1])]; - tensor inputs_21_pad_0 = const()[name = tensor("inputs_21_pad_0"), val = tensor([0, 0])]; - tensor inputs_21_dilations_0 = const()[name = tensor("inputs_21_dilations_0"), val = tensor([1])]; - tensor inputs_21_groups_0 = const()[name = tensor("inputs_21_groups_0"), val = tensor(1)]; - tensor input_53 = transpose(perm = input_53_perm_0, x = x_11)[name = tensor("transpose_68")]; - tensor inputs_21 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_21_dilations_0, groups = inputs_21_groups_0, pad = inputs_21_pad_0, pad_type = inputs_21_pad_type_0, strides = inputs_21_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_53)[name = tensor("inputs_21")]; - tensor var_530_split_sizes_0 = const()[name = tensor("op_530_split_sizes_0"), val = tensor([256, 256])]; - tensor var_530_axis_0 = const()[name = tensor("op_530_axis_0"), val = tensor(1)]; - tensor var_530_0, tensor var_530_1 = split(axis = var_530_axis_0, split_sizes = var_530_split_sizes_0, x = inputs_21)[name = tensor("op_530")]; - tensor var_532 = sigmoid(x = var_530_1)[name = tensor("op_532")]; - tensor inputs_23 = mul(x = var_530_0, y = var_532)[name = tensor("inputs_23")]; - tensor outputs_aug_7_pad_type_0 = const()[name = tensor("outputs_aug_7_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_7_pad_0 = const()[name = tensor("outputs_aug_7_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_7_groups_0 = const()[name = tensor("outputs_aug_7_groups_0"), val = tensor(256)]; - tensor outputs_aug_7_strides_0 = const()[name = tensor("outputs_aug_7_strides_0"), val = tensor([1])]; - tensor outputs_aug_7_dilations_0 = const()[name = tensor("outputs_aug_7_dilations_0"), val = tensor([1])]; - tensor outputs_aug_7 = conv(dilations = outputs_aug_7_dilations_0, groups = outputs_aug_7_groups_0, pad = outputs_aug_7_pad_0, pad_type = outputs_aug_7_pad_type_0, strides = outputs_aug_7_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_23)[name = tensor("outputs_aug_7")]; - tensor input_55_begin_0 = const()[name = tensor("input_55_begin_0"), val = tensor([0, 0, 0])]; - tensor input_55_end_0 = const()[name = tensor("input_55_end_0"), val = tensor([1, 256, 16])]; - tensor input_55_end_mask_0 = const()[name = tensor("input_55_end_mask_0"), val = tensor([true, true, false])]; - tensor input_55 = slice_by_index(begin = input_55_begin_0, end = input_55_end_0, end_mask = input_55_end_mask_0, x = outputs_aug_7)[name = tensor("input_55")]; - tensor inputs_25 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_55)[name = tensor("inputs_25")]; - tensor input_57 = silu(x = inputs_25)[name = tensor("input_57")]; - tensor input_59_pad_type_0 = const()[name = tensor("input_59_pad_type_0"), val = tensor("valid")]; - tensor input_59_strides_0 = const()[name = tensor("input_59_strides_0"), val = tensor([1])]; - tensor input_59_pad_0 = const()[name = tensor("input_59_pad_0"), val = tensor([0, 0])]; - tensor input_59_dilations_0 = const()[name = tensor("input_59_dilations_0"), val = tensor([1])]; - tensor input_59_groups_0 = const()[name = tensor("input_59_groups_0"), val = tensor(1)]; - tensor input_59 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_59_dilations_0, groups = input_59_groups_0, pad = input_59_pad_0, pad_type = input_59_pad_type_0, strides = input_59_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_57)[name = tensor("input_59")]; - tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_563_begin_0 = const()[name = tensor("op_563_begin_0"), val = tensor([0, -1, 0])]; - tensor var_563_end_0 = const()[name = tensor("op_563_end_0"), val = tensor([1, 16, 256])]; - tensor var_563_end_mask_0 = const()[name = tensor("op_563_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_59)[name = tensor("transpose_67")]; - tensor var_563 = slice_by_index(begin = var_563_begin_0, end = var_563_end_0, end_mask = var_563_end_mask_0, x = conv_out_7)[name = tensor("op_563")]; - tensor var_566_interleave_0 = const()[name = tensor("op_566_interleave_0"), val = tensor(false)]; - tensor var_566 = concat(axis = var_33, interleave = var_566_interleave_0, values = (var_311, var_395, var_479, var_563))[name = tensor("op_566")]; - tensor input_61 = add(x = x_3, y = var_566)[name = tensor("input_61")]; - tensor input_63_axes_0 = const()[name = tensor("input_63_axes_0"), val = tensor([-1])]; - tensor input_63 = layer_norm(axes = input_63_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_61)[name = tensor("input_63")]; - tensor inputs_27 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_63)[name = tensor("linear_8")]; - tensor input_65 = silu(x = inputs_27)[name = tensor("input_65")]; - tensor input_69 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_65)[name = tensor("linear_9")]; - tensor var_589 = const()[name = tensor("op_589"), val = tensor(0x1p-1)]; - tensor var_590 = mul(x = input_69, y = var_589)[name = tensor("op_590")]; - tensor input_71 = add(x = var_590, y = input_61)[name = tensor("input_71")]; - tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; - tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_71)[name = tensor("input_73")]; - tensor input_75_axes_0 = const()[name = tensor("input_75_axes_0"), val = tensor([-1])]; - tensor input_75 = layer_norm(axes = input_75_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_73)[name = tensor("input_75")]; - tensor inputs_29 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_75)[name = tensor("linear_10")]; - tensor input_77 = silu(x = inputs_29)[name = tensor("input_77")]; - tensor input_81 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_77)[name = tensor("linear_11")]; - tensor var_619 = const()[name = tensor("op_619"), val = tensor(0x1p-1)]; - tensor var_620 = mul(x = input_81, y = var_619)[name = tensor("op_620")]; - tensor input_83 = add(x = var_620, y = input_73)[name = tensor("input_83")]; - tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; - tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_83)[name = tensor("x_13")]; + tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_3_end_0 = const()[name = tensor("prev_kv_3_end_0"), val = tensor([2, 1, 4, 64, 64])]; tensor prev_kv_3_end_mask_0 = const()[name = tensor("prev_kv_3_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -636,389 +522,274 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_634 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_13)[name = tensor("linear_12")]; - tensor var_635 = const()[name = tensor("op_635"), val = tensor([1, 4, 4, 64])]; - tensor var_636 = reshape(shape = var_635, x = var_634)[name = tensor("op_636")]; + tensor var_473 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_474 = const()[name = tensor("op_474"), val = tensor([1, 4, 4, 64])]; + tensor var_475 = reshape(shape = var_474, x = var_473)[name = tensor("op_475")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_640 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_13)[name = tensor("linear_13")]; - tensor var_641 = const()[name = tensor("op_641"), val = tensor(0x1p-3)]; - tensor var_642 = mul(x = var_640, y = var_641)[name = tensor("op_642")]; - tensor var_643 = const()[name = tensor("op_643"), val = tensor([1, 4, 4, 64])]; - tensor var_644 = reshape(shape = var_643, x = var_642)[name = tensor("op_644")]; + tensor var_479 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_480 = const()[name = tensor("op_480"), val = tensor(0x1p-3)]; + tensor var_481 = mul(x = var_479, y = var_480)[name = tensor("op_481")]; + tensor var_482 = const()[name = tensor("op_482"), val = tensor([1, 4, 4, 64])]; + tensor var_483 = reshape(shape = var_482, x = var_481)[name = tensor("op_483")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_648 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_13)[name = tensor("linear_14")]; - tensor var_649 = const()[name = tensor("op_649"), val = tensor([1, 4, 4, 64])]; - tensor var_650 = reshape(shape = var_649, x = var_648)[name = tensor("op_650")]; + tensor var_487 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_488 = const()[name = tensor("op_488"), val = tensor([1, 4, 4, 64])]; + tensor var_489 = reshape(shape = var_488, x = var_487)[name = tensor("op_489")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_87 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_13)[name = tensor("linear_15")]; + tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; tensor s_t_3 = add(x = prev_scale_3, y = encoder__t_index)[name = tensor("s_t_3")]; tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_644)[name = tensor("transpose_65")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_636)[name = tensor("transpose_66")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_483)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_475)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_660 = const()[name = tensor("op_660"), val = tensor([4, 1])]; - tensor var_661 = reshape(shape = var_660, x = sqrt_s_t_3)[name = tensor("op_661")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_661)[name = tensor("M_3")]; - tensor var_663 = mul(x = qk_3, y = M_3)[name = tensor("op_663")]; + tensor var_499 = const()[name = tensor("op_499"), val = tensor([4, 1])]; + tensor var_500 = reshape(shape = var_499, x = sqrt_s_t_3)[name = tensor("op_500")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_500)[name = tensor("M_3")]; + tensor var_502 = mul(x = qk_3, y = M_3)[name = tensor("op_502")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_650)[name = tensor("transpose_64")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_663, y = v_3)[name = tensor("inner_3")]; - tensor var_665_transpose_x_0 = const()[name = tensor("op_665_transpose_x_0"), val = tensor(false)]; - tensor var_665_transpose_y_0 = const()[name = tensor("op_665_transpose_y_0"), val = tensor(false)]; - tensor var_665 = matmul(transpose_x = var_665_transpose_x_0, transpose_y = var_665_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_665")]; - tensor var_666 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_666")]; - tensor var_667 = const()[name = tensor("op_667"), val = tensor([1, 1, 4, 1])]; - tensor var_668 = reshape(shape = var_667, x = var_666)[name = tensor("op_668")]; - tensor cross_3 = mul(x = var_665, y = var_668)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_489)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_502, y = v_3)[name = tensor("inner_3")]; + tensor var_504_transpose_x_0 = const()[name = tensor("op_504_transpose_x_0"), val = tensor(false)]; + tensor var_504_transpose_y_0 = const()[name = tensor("op_504_transpose_y_0"), val = tensor(false)]; + tensor var_504 = matmul(transpose_x = var_504_transpose_x_0, transpose_y = var_504_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_504")]; + tensor var_505 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_505")]; + tensor var_506 = const()[name = tensor("op_506"), val = tensor([1, 1, 4, 1])]; + tensor var_507 = reshape(shape = var_506, x = var_505)[name = tensor("op_507")]; + tensor cross_3 = mul(x = var_504, y = var_507)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_671 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_671")]; - tensor var_673_transpose_x_1 = const()[name = tensor("op_673_transpose_x_1"), val = tensor(true)]; - tensor var_673_transpose_y_1 = const()[name = tensor("op_673_transpose_y_1"), val = tensor(false)]; - tensor var_673 = matmul(transpose_x = var_673_transpose_x_1, transpose_y = var_673_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_673")]; - tensor new_kv_unnorm_3 = add(x = var_671, y = var_673)[name = tensor("new_kv_unnorm_3")]; - tensor var_675 = const()[name = tensor("op_675"), val = tensor(0x1p+2)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_675)[name = tensor("new_scale_3")]; - tensor var_677 = sqrt(x = new_scale_3)[name = tensor("op_677")]; - tensor var_678 = real_div(x = new_kv_unnorm_3, y = var_677)[name = tensor("op_678")]; - tensor var_679_perm_0 = const()[name = tensor("op_679_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_510 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_510")]; + tensor var_512_transpose_x_1 = const()[name = tensor("op_512_transpose_x_1"), val = tensor(true)]; + tensor var_512_transpose_y_1 = const()[name = tensor("op_512_transpose_y_1"), val = tensor(false)]; + tensor var_512 = matmul(transpose_x = var_512_transpose_x_1, transpose_y = var_512_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_512")]; + tensor new_kv_unnorm_3 = add(x = var_510, y = var_512)[name = tensor("new_kv_unnorm_3")]; + tensor var_514 = const()[name = tensor("op_514"), val = tensor(0x1p+2)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_514)[name = tensor("new_scale_3")]; + tensor var_516 = sqrt(x = new_scale_3)[name = tensor("op_516")]; + tensor var_517 = real_div(x = new_kv_unnorm_3, y = var_516)[name = tensor("op_517")]; + tensor var_518_perm_0 = const()[name = tensor("op_518_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_679 = transpose(perm = var_679_perm_0, x = out_7)[name = tensor("transpose_63")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_679)[name = tensor("out_9")]; - tensor var_683 = const()[name = tensor("op_683"), val = tensor([1, 4, 256])]; - tensor out_11 = reshape(shape = var_683, x = out_9)[name = tensor("out_11")]; - tensor var_685 = silu(x = input_87)[name = tensor("op_685")]; - tensor input_89 = mul(x = var_685, y = out_11)[name = tensor("input_89")]; - tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_89)[name = tensor("linear_16")]; - tensor x_15 = add(x = input_83, y = ret_out_3)[name = tensor("x_15")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor var_518 = transpose(perm = var_518_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_518)[name = tensor("out_9")]; + tensor var_522 = const()[name = tensor("op_522"), val = tensor([1, 4, 256])]; + tensor out_11 = reshape(shape = var_522, x = out_9)[name = tensor("out_11")]; + tensor var_524 = silu(x = input_57)[name = tensor("op_524")]; + tensor input_59 = mul(x = var_524, y = out_11)[name = tensor("input_59")]; + tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; + tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; + tensor window_11_begin_0 = const()[name = tensor("window_11_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_11_end_0 = const()[name = tensor("window_11_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_11_end_mask_0 = const()[name = tensor("window_11_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_11_squeeze_mask_0 = const()[name = tensor("window_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_11 = slice_by_index(begin = window_11_begin_0, end = window_11_end_0, end_mask = window_11_end_mask_0, squeeze_mask = window_11_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_11")]; tensor filled_9_begin_0 = const()[name = tensor("filled_9_begin_0"), val = tensor([1, 0])]; tensor filled_9_end_0 = const()[name = tensor("filled_9_end_0"), val = tensor([2, 1])]; tensor filled_9_end_mask_0 = const()[name = tensor("filled_9_end_mask_0"), val = tensor([false, true])]; tensor filled_9_squeeze_mask_0 = const()[name = tensor("filled_9_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_9 = slice_by_index(begin = filled_9_begin_0, end = filled_9_end_0, end_mask = filled_9_end_mask_0, squeeze_mask = filled_9_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_9")]; - tensor var_694_begin_0 = const()[name = tensor("op_694_begin_0"), val = tensor([0, 0, 0])]; - tensor var_694_end_0 = const()[name = tensor("op_694_end_0"), val = tensor([1, 1, 256])]; - tensor var_694_end_mask_0 = const()[name = tensor("op_694_end_mask_0"), val = tensor([true, false, true])]; - tensor var_694 = slice_by_index(begin = var_694_begin_0, end = var_694_end_0, end_mask = var_694_end_mask_0, x = x_15)[name = tensor("op_694")]; - tensor var_697_begin_0 = const()[name = tensor("op_697_begin_0"), val = tensor([0, 1, 0])]; - tensor var_697_end_0 = const()[name = tensor("op_697_end_0"), val = tensor([1, 16, 256])]; - tensor var_697_end_mask_0 = const()[name = tensor("op_697_end_mask_0"), val = tensor([true, true, true])]; - tensor var_697 = slice_by_index(begin = var_697_begin_0, end = var_697_end_0, end_mask = var_697_end_mask_0, x = window_3)[name = tensor("op_697")]; + tensor var_533_begin_0 = const()[name = tensor("op_533_begin_0"), val = tensor([0, 0, 0])]; + tensor var_533_end_0 = const()[name = tensor("op_533_end_0"), val = tensor([1, 1, 256])]; + tensor var_533_end_mask_0 = const()[name = tensor("op_533_end_mask_0"), val = tensor([true, false, true])]; + tensor var_533 = slice_by_index(begin = var_533_begin_0, end = var_533_end_0, end_mask = var_533_end_mask_0, x = x_9)[name = tensor("op_533")]; + tensor var_536_begin_0 = const()[name = tensor("op_536_begin_0"), val = tensor([0, 1, 0])]; + tensor var_536_end_0 = const()[name = tensor("op_536_end_0"), val = tensor([1, 16, 256])]; + tensor var_536_end_mask_0 = const()[name = tensor("op_536_end_mask_0"), val = tensor([true, true, true])]; + tensor var_536 = slice_by_index(begin = var_536_begin_0, end = var_536_end_0, end_mask = var_536_end_mask_0, x = window_11)[name = tensor("op_536")]; tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; - tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_697, var_694))[name = tensor("rolled_9")]; + tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_536, var_533))[name = tensor("rolled_9")]; tensor clip_8 = clip(alpha = var_21, beta = var_20, x = filled_9)[name = tensor("clip_8")]; - tensor var_702 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_702")]; - tensor var_703_dtype_0 = const()[name = tensor("op_703_dtype_0"), val = tensor("fp32")]; - tensor var_704 = const()[name = tensor("op_704"), val = tensor([1, 16, 1])]; - tensor var_703 = cast(dtype = var_703_dtype_0, x = var_702)[name = tensor("cast_167")]; - tensor fill_mask_9 = reshape(shape = var_704, x = var_703)[name = tensor("fill_mask_9")]; + tensor var_541 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_541")]; + tensor var_542_dtype_0 = const()[name = tensor("op_542_dtype_0"), val = tensor("fp32")]; + tensor var_543 = const()[name = tensor("op_543"), val = tensor([1, 16, 1])]; + tensor var_542 = cast(dtype = var_542_dtype_0, x = var_541)[name = tensor("cast_167")]; + tensor fill_mask_9 = reshape(shape = var_543, x = var_542)[name = tensor("fill_mask_9")]; tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_694)[name = tensor("x_expanded_9")]; - tensor var_708 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_708")]; - tensor var_709 = mul(x = window_3, y = var_708)[name = tensor("op_709")]; - tensor var_710 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_710")]; - tensor filled_window_9 = add(x = var_709, y = var_710)[name = tensor("filled_window_9")]; - tensor var_712 = greater_equal(x = filled_9, y = var_16)[name = tensor("op_712")]; - tensor var_713_dtype_0 = const()[name = tensor("op_713_dtype_0"), val = tensor("fp32")]; - tensor var_714 = const()[name = tensor("op_714"), val = tensor([1, 1, 1])]; - tensor var_713 = cast(dtype = var_713_dtype_0, x = var_712)[name = tensor("cast_166")]; - tensor use_roll_9 = reshape(shape = var_714, x = var_713)[name = tensor("use_roll_9")]; - tensor var_716 = sub(x = var_17, y = use_roll_9)[name = tensor("op_716")]; - tensor var_717 = mul(x = filled_window_9, y = var_716)[name = tensor("op_717")]; - tensor var_718 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_718")]; - tensor input_91 = add(x = var_717, y = var_718)[name = tensor("input_91")]; - tensor var_720 = const()[name = tensor("op_720"), val = tensor(0x1p+0)]; - tensor var_721 = add(x = filled_9, y = var_720)[name = tensor("op_721")]; - tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_721)[name = tensor("clip_9")]; - tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; - tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_91)[name = tensor("x_17")]; - tensor input_93_perm_0 = const()[name = tensor("input_93_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_31_pad_type_0 = const()[name = tensor("inputs_31_pad_type_0"), val = tensor("valid")]; - tensor inputs_31_strides_0 = const()[name = tensor("inputs_31_strides_0"), val = tensor([1])]; - tensor inputs_31_pad_0 = const()[name = tensor("inputs_31_pad_0"), val = tensor([0, 0])]; - tensor inputs_31_dilations_0 = const()[name = tensor("inputs_31_dilations_0"), val = tensor([1])]; - tensor inputs_31_groups_0 = const()[name = tensor("inputs_31_groups_0"), val = tensor(1)]; - tensor input_93 = transpose(perm = input_93_perm_0, x = x_17)[name = tensor("transpose_62")]; - tensor inputs_31 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_31_dilations_0, groups = inputs_31_groups_0, pad = inputs_31_pad_0, pad_type = inputs_31_pad_type_0, strides = inputs_31_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_93)[name = tensor("inputs_31")]; - tensor var_742_split_sizes_0 = const()[name = tensor("op_742_split_sizes_0"), val = tensor([256, 256])]; - tensor var_742_axis_0 = const()[name = tensor("op_742_axis_0"), val = tensor(1)]; - tensor var_742_0, tensor var_742_1 = split(axis = var_742_axis_0, split_sizes = var_742_split_sizes_0, x = inputs_31)[name = tensor("op_742")]; - tensor var_744 = sigmoid(x = var_742_1)[name = tensor("op_744")]; - tensor inputs_33 = mul(x = var_742_0, y = var_744)[name = tensor("inputs_33")]; - tensor outputs_aug_9_pad_type_0 = const()[name = tensor("outputs_aug_9_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_9_pad_0 = const()[name = tensor("outputs_aug_9_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_9_groups_0 = const()[name = tensor("outputs_aug_9_groups_0"), val = tensor(256)]; - tensor outputs_aug_9_strides_0 = const()[name = tensor("outputs_aug_9_strides_0"), val = tensor([1])]; - tensor outputs_aug_9_dilations_0 = const()[name = tensor("outputs_aug_9_dilations_0"), val = tensor([1])]; - tensor outputs_aug_9 = conv(dilations = outputs_aug_9_dilations_0, groups = outputs_aug_9_groups_0, pad = outputs_aug_9_pad_0, pad_type = outputs_aug_9_pad_type_0, strides = outputs_aug_9_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_33)[name = tensor("outputs_aug_9")]; - tensor input_95_begin_0 = const()[name = tensor("input_95_begin_0"), val = tensor([0, 0, 0])]; - tensor input_95_end_0 = const()[name = tensor("input_95_end_0"), val = tensor([1, 256, 16])]; - tensor input_95_end_mask_0 = const()[name = tensor("input_95_end_mask_0"), val = tensor([true, true, false])]; - tensor input_95 = slice_by_index(begin = input_95_begin_0, end = input_95_end_0, end_mask = input_95_end_mask_0, x = outputs_aug_9)[name = tensor("input_95")]; - tensor inputs_35 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_95)[name = tensor("inputs_35")]; - tensor input_97 = silu(x = inputs_35)[name = tensor("input_97")]; - tensor input_99_pad_type_0 = const()[name = tensor("input_99_pad_type_0"), val = tensor("valid")]; - tensor input_99_strides_0 = const()[name = tensor("input_99_strides_0"), val = tensor([1])]; - tensor input_99_pad_0 = const()[name = tensor("input_99_pad_0"), val = tensor([0, 0])]; - tensor input_99_dilations_0 = const()[name = tensor("input_99_dilations_0"), val = tensor([1])]; - tensor input_99_groups_0 = const()[name = tensor("input_99_groups_0"), val = tensor(1)]; - tensor input_99 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_99_dilations_0, groups = input_99_groups_0, pad = input_99_pad_0, pad_type = input_99_pad_type_0, strides = input_99_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_97)[name = tensor("input_99")]; - tensor conv_out_9_perm_0 = const()[name = tensor("conv_out_9_perm_0"), val = tensor([0, 2, 1])]; - tensor var_775_begin_0 = const()[name = tensor("op_775_begin_0"), val = tensor([0, -1, 0])]; - tensor var_775_end_0 = const()[name = tensor("op_775_end_0"), val = tensor([1, 16, 256])]; - tensor var_775_end_mask_0 = const()[name = tensor("op_775_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_9 = transpose(perm = conv_out_9_perm_0, x = input_99)[name = tensor("transpose_61")]; - tensor var_775 = slice_by_index(begin = var_775_begin_0, end = var_775_end_0, end_mask = var_775_end_mask_0, x = conv_out_9)[name = tensor("op_775")]; - tensor var_778_begin_0 = const()[name = tensor("op_778_begin_0"), val = tensor([0, 1, 0])]; - tensor var_778_end_0 = const()[name = tensor("op_778_end_0"), val = tensor([1, 2, 256])]; - tensor var_778_end_mask_0 = const()[name = tensor("op_778_end_mask_0"), val = tensor([true, false, true])]; - tensor var_778 = slice_by_index(begin = var_778_begin_0, end = var_778_end_0, end_mask = var_778_end_mask_0, x = x_15)[name = tensor("op_778")]; - tensor var_781_begin_0 = const()[name = tensor("op_781_begin_0"), val = tensor([0, 1, 0])]; - tensor var_781_end_0 = const()[name = tensor("op_781_end_0"), val = tensor([1, 16, 256])]; - tensor var_781_end_mask_0 = const()[name = tensor("op_781_end_mask_0"), val = tensor([true, true, true])]; - tensor var_781 = slice_by_index(begin = var_781_begin_0, end = var_781_end_0, end_mask = var_781_end_mask_0, x = input_91)[name = tensor("op_781")]; + tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_533)[name = tensor("x_expanded_9")]; + tensor var_547 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_547")]; + tensor var_548 = mul(x = window_11, y = var_547)[name = tensor("op_548")]; + tensor var_549 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_549")]; + tensor filled_window_9 = add(x = var_548, y = var_549)[name = tensor("filled_window_9")]; + tensor var_551 = greater_equal(x = filled_9, y = var_16)[name = tensor("op_551")]; + tensor var_552_dtype_0 = const()[name = tensor("op_552_dtype_0"), val = tensor("fp32")]; + tensor var_553 = const()[name = tensor("op_553"), val = tensor([1, 1, 1])]; + tensor var_552 = cast(dtype = var_552_dtype_0, x = var_551)[name = tensor("cast_166")]; + tensor use_roll_9 = reshape(shape = var_553, x = var_552)[name = tensor("use_roll_9")]; + tensor var_555 = sub(x = var_17, y = use_roll_9)[name = tensor("op_555")]; + tensor var_556 = mul(x = filled_window_9, y = var_555)[name = tensor("op_556")]; + tensor var_557 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_557")]; + tensor window_13 = add(x = var_556, y = var_557)[name = tensor("window_13")]; + tensor var_559 = const()[name = tensor("op_559"), val = tensor(0x1p+0)]; + tensor var_560 = add(x = filled_9, y = var_559)[name = tensor("op_560")]; + tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_560)[name = tensor("clip_9")]; + tensor var_563_begin_0 = const()[name = tensor("op_563_begin_0"), val = tensor([0, 1, 0])]; + tensor var_563_end_0 = const()[name = tensor("op_563_end_0"), val = tensor([1, 2, 256])]; + tensor var_563_end_mask_0 = const()[name = tensor("op_563_end_mask_0"), val = tensor([true, false, true])]; + tensor var_563 = slice_by_index(begin = var_563_begin_0, end = var_563_end_0, end_mask = var_563_end_mask_0, x = x_9)[name = tensor("op_563")]; + tensor var_566_begin_0 = const()[name = tensor("op_566_begin_0"), val = tensor([0, 1, 0])]; + tensor var_566_end_0 = const()[name = tensor("op_566_end_0"), val = tensor([1, 16, 256])]; + tensor var_566_end_mask_0 = const()[name = tensor("op_566_end_mask_0"), val = tensor([true, true, true])]; + tensor var_566 = slice_by_index(begin = var_566_begin_0, end = var_566_end_0, end_mask = var_566_end_mask_0, x = window_13)[name = tensor("op_566")]; tensor rolled_11_interleave_0 = const()[name = tensor("rolled_11_interleave_0"), val = tensor(false)]; - tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_781, var_778))[name = tensor("rolled_11")]; + tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_566, var_563))[name = tensor("rolled_11")]; tensor clip_10 = clip(alpha = var_21, beta = var_20, x = clip_9)[name = tensor("clip_10")]; - tensor var_786 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_786")]; - tensor var_787_dtype_0 = const()[name = tensor("op_787_dtype_0"), val = tensor("fp32")]; - tensor var_788 = const()[name = tensor("op_788"), val = tensor([1, 16, 1])]; - tensor var_787 = cast(dtype = var_787_dtype_0, x = var_786)[name = tensor("cast_165")]; - tensor fill_mask_11 = reshape(shape = var_788, x = var_787)[name = tensor("fill_mask_11")]; + tensor var_571 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_571")]; + tensor var_572_dtype_0 = const()[name = tensor("op_572_dtype_0"), val = tensor("fp32")]; + tensor var_573 = const()[name = tensor("op_573"), val = tensor([1, 16, 1])]; + tensor var_572 = cast(dtype = var_572_dtype_0, x = var_571)[name = tensor("cast_165")]; + tensor fill_mask_11 = reshape(shape = var_573, x = var_572)[name = tensor("fill_mask_11")]; tensor x_expanded_11_reps_0 = const()[name = tensor("x_expanded_11_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_778)[name = tensor("x_expanded_11")]; - tensor var_792 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_792")]; - tensor var_793 = mul(x = input_91, y = var_792)[name = tensor("op_793")]; - tensor var_794 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_794")]; - tensor filled_window_11 = add(x = var_793, y = var_794)[name = tensor("filled_window_11")]; - tensor var_796 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_796")]; - tensor var_797_dtype_0 = const()[name = tensor("op_797_dtype_0"), val = tensor("fp32")]; - tensor var_798 = const()[name = tensor("op_798"), val = tensor([1, 1, 1])]; - tensor var_797 = cast(dtype = var_797_dtype_0, x = var_796)[name = tensor("cast_164")]; - tensor use_roll_11 = reshape(shape = var_798, x = var_797)[name = tensor("use_roll_11")]; - tensor var_800 = sub(x = var_17, y = use_roll_11)[name = tensor("op_800")]; - tensor var_801 = mul(x = filled_window_11, y = var_800)[name = tensor("op_801")]; - tensor var_802 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_802")]; - tensor input_101 = add(x = var_801, y = var_802)[name = tensor("input_101")]; - tensor var_804 = const()[name = tensor("op_804"), val = tensor(0x1p+0)]; - tensor var_805 = add(x = clip_9, y = var_804)[name = tensor("op_805")]; - tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_805)[name = tensor("clip_11")]; - tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; - tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_101)[name = tensor("x_19")]; - tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_37_pad_type_0 = const()[name = tensor("inputs_37_pad_type_0"), val = tensor("valid")]; - tensor inputs_37_strides_0 = const()[name = tensor("inputs_37_strides_0"), val = tensor([1])]; - tensor inputs_37_pad_0 = const()[name = tensor("inputs_37_pad_0"), val = tensor([0, 0])]; - tensor inputs_37_dilations_0 = const()[name = tensor("inputs_37_dilations_0"), val = tensor([1])]; - tensor inputs_37_groups_0 = const()[name = tensor("inputs_37_groups_0"), val = tensor(1)]; - tensor input_103 = transpose(perm = input_103_perm_0, x = x_19)[name = tensor("transpose_60")]; - tensor inputs_37 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_37_dilations_0, groups = inputs_37_groups_0, pad = inputs_37_pad_0, pad_type = inputs_37_pad_type_0, strides = inputs_37_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_37")]; - tensor var_826_split_sizes_0 = const()[name = tensor("op_826_split_sizes_0"), val = tensor([256, 256])]; - tensor var_826_axis_0 = const()[name = tensor("op_826_axis_0"), val = tensor(1)]; - tensor var_826_0, tensor var_826_1 = split(axis = var_826_axis_0, split_sizes = var_826_split_sizes_0, x = inputs_37)[name = tensor("op_826")]; - tensor var_828 = sigmoid(x = var_826_1)[name = tensor("op_828")]; - tensor inputs_39 = mul(x = var_826_0, y = var_828)[name = tensor("inputs_39")]; - tensor outputs_aug_11_pad_type_0 = const()[name = tensor("outputs_aug_11_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_11_pad_0 = const()[name = tensor("outputs_aug_11_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_11_groups_0 = const()[name = tensor("outputs_aug_11_groups_0"), val = tensor(256)]; - tensor outputs_aug_11_strides_0 = const()[name = tensor("outputs_aug_11_strides_0"), val = tensor([1])]; - tensor outputs_aug_11_dilations_0 = const()[name = tensor("outputs_aug_11_dilations_0"), val = tensor([1])]; - tensor outputs_aug_11 = conv(dilations = outputs_aug_11_dilations_0, groups = outputs_aug_11_groups_0, pad = outputs_aug_11_pad_0, pad_type = outputs_aug_11_pad_type_0, strides = outputs_aug_11_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_39)[name = tensor("outputs_aug_11")]; - tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; - tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([1, 256, 16])]; - tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; - tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_11)[name = tensor("input_105")]; - tensor inputs_41 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_105)[name = tensor("inputs_41")]; - tensor input_107 = silu(x = inputs_41)[name = tensor("input_107")]; - tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; - tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; - tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; - tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; - tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; - tensor input_109 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; - tensor conv_out_11_perm_0 = const()[name = tensor("conv_out_11_perm_0"), val = tensor([0, 2, 1])]; - tensor var_859_begin_0 = const()[name = tensor("op_859_begin_0"), val = tensor([0, -1, 0])]; - tensor var_859_end_0 = const()[name = tensor("op_859_end_0"), val = tensor([1, 16, 256])]; - tensor var_859_end_mask_0 = const()[name = tensor("op_859_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_11 = transpose(perm = conv_out_11_perm_0, x = input_109)[name = tensor("transpose_59")]; - tensor var_859 = slice_by_index(begin = var_859_begin_0, end = var_859_end_0, end_mask = var_859_end_mask_0, x = conv_out_11)[name = tensor("op_859")]; - tensor var_862_begin_0 = const()[name = tensor("op_862_begin_0"), val = tensor([0, 2, 0])]; - tensor var_862_end_0 = const()[name = tensor("op_862_end_0"), val = tensor([1, 3, 256])]; - tensor var_862_end_mask_0 = const()[name = tensor("op_862_end_mask_0"), val = tensor([true, false, true])]; - tensor var_862 = slice_by_index(begin = var_862_begin_0, end = var_862_end_0, end_mask = var_862_end_mask_0, x = x_15)[name = tensor("op_862")]; - tensor var_865_begin_0 = const()[name = tensor("op_865_begin_0"), val = tensor([0, 1, 0])]; - tensor var_865_end_0 = const()[name = tensor("op_865_end_0"), val = tensor([1, 16, 256])]; - tensor var_865_end_mask_0 = const()[name = tensor("op_865_end_mask_0"), val = tensor([true, true, true])]; - tensor var_865 = slice_by_index(begin = var_865_begin_0, end = var_865_end_0, end_mask = var_865_end_mask_0, x = input_101)[name = tensor("op_865")]; + tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_563)[name = tensor("x_expanded_11")]; + tensor var_577 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_577")]; + tensor var_578 = mul(x = window_13, y = var_577)[name = tensor("op_578")]; + tensor var_579 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_579")]; + tensor filled_window_11 = add(x = var_578, y = var_579)[name = tensor("filled_window_11")]; + tensor var_581 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_581")]; + tensor var_582_dtype_0 = const()[name = tensor("op_582_dtype_0"), val = tensor("fp32")]; + tensor var_583 = const()[name = tensor("op_583"), val = tensor([1, 1, 1])]; + tensor var_582 = cast(dtype = var_582_dtype_0, x = var_581)[name = tensor("cast_164")]; + tensor use_roll_11 = reshape(shape = var_583, x = var_582)[name = tensor("use_roll_11")]; + tensor var_585 = sub(x = var_17, y = use_roll_11)[name = tensor("op_585")]; + tensor var_586 = mul(x = filled_window_11, y = var_585)[name = tensor("op_586")]; + tensor var_587 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_587")]; + tensor window_15 = add(x = var_586, y = var_587)[name = tensor("window_15")]; + tensor var_589 = const()[name = tensor("op_589"), val = tensor(0x1p+0)]; + tensor var_590 = add(x = clip_9, y = var_589)[name = tensor("op_590")]; + tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_590)[name = tensor("clip_11")]; + tensor var_593_begin_0 = const()[name = tensor("op_593_begin_0"), val = tensor([0, 2, 0])]; + tensor var_593_end_0 = const()[name = tensor("op_593_end_0"), val = tensor([1, 3, 256])]; + tensor var_593_end_mask_0 = const()[name = tensor("op_593_end_mask_0"), val = tensor([true, false, true])]; + tensor var_593 = slice_by_index(begin = var_593_begin_0, end = var_593_end_0, end_mask = var_593_end_mask_0, x = x_9)[name = tensor("op_593")]; + tensor var_596_begin_0 = const()[name = tensor("op_596_begin_0"), val = tensor([0, 1, 0])]; + tensor var_596_end_0 = const()[name = tensor("op_596_end_0"), val = tensor([1, 16, 256])]; + tensor var_596_end_mask_0 = const()[name = tensor("op_596_end_mask_0"), val = tensor([true, true, true])]; + tensor var_596 = slice_by_index(begin = var_596_begin_0, end = var_596_end_0, end_mask = var_596_end_mask_0, x = window_15)[name = tensor("op_596")]; tensor rolled_13_interleave_0 = const()[name = tensor("rolled_13_interleave_0"), val = tensor(false)]; - tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_865, var_862))[name = tensor("rolled_13")]; + tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_596, var_593))[name = tensor("rolled_13")]; tensor clip_12 = clip(alpha = var_21, beta = var_20, x = clip_11)[name = tensor("clip_12")]; - tensor var_870 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_870")]; - tensor var_871_dtype_0 = const()[name = tensor("op_871_dtype_0"), val = tensor("fp32")]; - tensor var_872 = const()[name = tensor("op_872"), val = tensor([1, 16, 1])]; - tensor var_871 = cast(dtype = var_871_dtype_0, x = var_870)[name = tensor("cast_163")]; - tensor fill_mask_13 = reshape(shape = var_872, x = var_871)[name = tensor("fill_mask_13")]; + tensor var_601 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_601")]; + tensor var_602_dtype_0 = const()[name = tensor("op_602_dtype_0"), val = tensor("fp32")]; + tensor var_603 = const()[name = tensor("op_603"), val = tensor([1, 16, 1])]; + tensor var_602 = cast(dtype = var_602_dtype_0, x = var_601)[name = tensor("cast_163")]; + tensor fill_mask_13 = reshape(shape = var_603, x = var_602)[name = tensor("fill_mask_13")]; tensor x_expanded_13_reps_0 = const()[name = tensor("x_expanded_13_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_862)[name = tensor("x_expanded_13")]; - tensor var_876 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_876")]; - tensor var_877 = mul(x = input_101, y = var_876)[name = tensor("op_877")]; - tensor var_878 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_878")]; - tensor filled_window_13 = add(x = var_877, y = var_878)[name = tensor("filled_window_13")]; - tensor var_880 = greater_equal(x = clip_11, y = var_16)[name = tensor("op_880")]; - tensor var_881_dtype_0 = const()[name = tensor("op_881_dtype_0"), val = tensor("fp32")]; - tensor var_882 = const()[name = tensor("op_882"), val = tensor([1, 1, 1])]; - tensor var_881 = cast(dtype = var_881_dtype_0, x = var_880)[name = tensor("cast_162")]; - tensor use_roll_13 = reshape(shape = var_882, x = var_881)[name = tensor("use_roll_13")]; - tensor var_884 = sub(x = var_17, y = use_roll_13)[name = tensor("op_884")]; - tensor var_885 = mul(x = filled_window_13, y = var_884)[name = tensor("op_885")]; - tensor var_886 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_886")]; - tensor input_111 = add(x = var_885, y = var_886)[name = tensor("input_111")]; - tensor var_888 = const()[name = tensor("op_888"), val = tensor(0x1p+0)]; - tensor var_889 = add(x = clip_11, y = var_888)[name = tensor("op_889")]; - tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_889)[name = tensor("clip_13")]; - tensor x_21_axes_0 = const()[name = tensor("x_21_axes_0"), val = tensor([-1])]; - tensor x_21 = layer_norm(axes = x_21_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_111)[name = tensor("x_21")]; - tensor input_113_perm_0 = const()[name = tensor("input_113_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_43_pad_type_0 = const()[name = tensor("inputs_43_pad_type_0"), val = tensor("valid")]; - tensor inputs_43_strides_0 = const()[name = tensor("inputs_43_strides_0"), val = tensor([1])]; - tensor inputs_43_pad_0 = const()[name = tensor("inputs_43_pad_0"), val = tensor([0, 0])]; - tensor inputs_43_dilations_0 = const()[name = tensor("inputs_43_dilations_0"), val = tensor([1])]; - tensor inputs_43_groups_0 = const()[name = tensor("inputs_43_groups_0"), val = tensor(1)]; - tensor input_113 = transpose(perm = input_113_perm_0, x = x_21)[name = tensor("transpose_58")]; - tensor inputs_43 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_43_dilations_0, groups = inputs_43_groups_0, pad = inputs_43_pad_0, pad_type = inputs_43_pad_type_0, strides = inputs_43_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_113)[name = tensor("inputs_43")]; - tensor var_910_split_sizes_0 = const()[name = tensor("op_910_split_sizes_0"), val = tensor([256, 256])]; - tensor var_910_axis_0 = const()[name = tensor("op_910_axis_0"), val = tensor(1)]; - tensor var_910_0, tensor var_910_1 = split(axis = var_910_axis_0, split_sizes = var_910_split_sizes_0, x = inputs_43)[name = tensor("op_910")]; - tensor var_912 = sigmoid(x = var_910_1)[name = tensor("op_912")]; - tensor inputs_45 = mul(x = var_910_0, y = var_912)[name = tensor("inputs_45")]; - tensor outputs_aug_13_pad_type_0 = const()[name = tensor("outputs_aug_13_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_13_pad_0 = const()[name = tensor("outputs_aug_13_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_13_groups_0 = const()[name = tensor("outputs_aug_13_groups_0"), val = tensor(256)]; - tensor outputs_aug_13_strides_0 = const()[name = tensor("outputs_aug_13_strides_0"), val = tensor([1])]; - tensor outputs_aug_13_dilations_0 = const()[name = tensor("outputs_aug_13_dilations_0"), val = tensor([1])]; - tensor outputs_aug_13 = conv(dilations = outputs_aug_13_dilations_0, groups = outputs_aug_13_groups_0, pad = outputs_aug_13_pad_0, pad_type = outputs_aug_13_pad_type_0, strides = outputs_aug_13_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_45)[name = tensor("outputs_aug_13")]; - tensor input_115_begin_0 = const()[name = tensor("input_115_begin_0"), val = tensor([0, 0, 0])]; - tensor input_115_end_0 = const()[name = tensor("input_115_end_0"), val = tensor([1, 256, 16])]; - tensor input_115_end_mask_0 = const()[name = tensor("input_115_end_mask_0"), val = tensor([true, true, false])]; - tensor input_115 = slice_by_index(begin = input_115_begin_0, end = input_115_end_0, end_mask = input_115_end_mask_0, x = outputs_aug_13)[name = tensor("input_115")]; - tensor inputs_47 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_115)[name = tensor("inputs_47")]; - tensor input_117 = silu(x = inputs_47)[name = tensor("input_117")]; - tensor input_119_pad_type_0 = const()[name = tensor("input_119_pad_type_0"), val = tensor("valid")]; - tensor input_119_strides_0 = const()[name = tensor("input_119_strides_0"), val = tensor([1])]; - tensor input_119_pad_0 = const()[name = tensor("input_119_pad_0"), val = tensor([0, 0])]; - tensor input_119_dilations_0 = const()[name = tensor("input_119_dilations_0"), val = tensor([1])]; - tensor input_119_groups_0 = const()[name = tensor("input_119_groups_0"), val = tensor(1)]; - tensor input_119 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_119_dilations_0, groups = input_119_groups_0, pad = input_119_pad_0, pad_type = input_119_pad_type_0, strides = input_119_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_117)[name = tensor("input_119")]; - tensor conv_out_13_perm_0 = const()[name = tensor("conv_out_13_perm_0"), val = tensor([0, 2, 1])]; - tensor var_943_begin_0 = const()[name = tensor("op_943_begin_0"), val = tensor([0, -1, 0])]; - tensor var_943_end_0 = const()[name = tensor("op_943_end_0"), val = tensor([1, 16, 256])]; - tensor var_943_end_mask_0 = const()[name = tensor("op_943_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_13 = transpose(perm = conv_out_13_perm_0, x = input_119)[name = tensor("transpose_57")]; - tensor var_943 = slice_by_index(begin = var_943_begin_0, end = var_943_end_0, end_mask = var_943_end_mask_0, x = conv_out_13)[name = tensor("op_943")]; - tensor var_946_begin_0 = const()[name = tensor("op_946_begin_0"), val = tensor([0, 3, 0])]; - tensor var_946_end_0 = const()[name = tensor("op_946_end_0"), val = tensor([1, 1, 256])]; - tensor var_946_end_mask_0 = const()[name = tensor("op_946_end_mask_0"), val = tensor([true, true, true])]; - tensor var_946 = slice_by_index(begin = var_946_begin_0, end = var_946_end_0, end_mask = var_946_end_mask_0, x = x_15)[name = tensor("op_946")]; - tensor var_949_begin_0 = const()[name = tensor("op_949_begin_0"), val = tensor([0, 1, 0])]; - tensor var_949_end_0 = const()[name = tensor("op_949_end_0"), val = tensor([1, 16, 256])]; - tensor var_949_end_mask_0 = const()[name = tensor("op_949_end_mask_0"), val = tensor([true, true, true])]; - tensor var_949 = slice_by_index(begin = var_949_begin_0, end = var_949_end_0, end_mask = var_949_end_mask_0, x = input_111)[name = tensor("op_949")]; + tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_593)[name = tensor("x_expanded_13")]; + tensor var_607 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_607")]; + tensor var_608 = mul(x = window_15, y = var_607)[name = tensor("op_608")]; + tensor var_609 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_609")]; + tensor filled_window_13 = add(x = var_608, y = var_609)[name = tensor("filled_window_13")]; + tensor var_611 = greater_equal(x = clip_11, y = var_16)[name = tensor("op_611")]; + tensor var_612_dtype_0 = const()[name = tensor("op_612_dtype_0"), val = tensor("fp32")]; + tensor var_613 = const()[name = tensor("op_613"), val = tensor([1, 1, 1])]; + tensor var_612 = cast(dtype = var_612_dtype_0, x = var_611)[name = tensor("cast_162")]; + tensor use_roll_13 = reshape(shape = var_613, x = var_612)[name = tensor("use_roll_13")]; + tensor var_615 = sub(x = var_17, y = use_roll_13)[name = tensor("op_615")]; + tensor var_616 = mul(x = filled_window_13, y = var_615)[name = tensor("op_616")]; + tensor var_617 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_617")]; + tensor window_17 = add(x = var_616, y = var_617)[name = tensor("window_17")]; + tensor var_619 = const()[name = tensor("op_619"), val = tensor(0x1p+0)]; + tensor var_620 = add(x = clip_11, y = var_619)[name = tensor("op_620")]; + tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_620)[name = tensor("clip_13")]; + tensor var_623_begin_0 = const()[name = tensor("op_623_begin_0"), val = tensor([0, 3, 0])]; + tensor var_623_end_0 = const()[name = tensor("op_623_end_0"), val = tensor([1, 1, 256])]; + tensor var_623_end_mask_0 = const()[name = tensor("op_623_end_mask_0"), val = tensor([true, true, true])]; + tensor var_623 = slice_by_index(begin = var_623_begin_0, end = var_623_end_0, end_mask = var_623_end_mask_0, x = x_9)[name = tensor("op_623")]; + tensor var_626_begin_0 = const()[name = tensor("op_626_begin_0"), val = tensor([0, 1, 0])]; + tensor var_626_end_0 = const()[name = tensor("op_626_end_0"), val = tensor([1, 16, 256])]; + tensor var_626_end_mask_0 = const()[name = tensor("op_626_end_mask_0"), val = tensor([true, true, true])]; + tensor var_626 = slice_by_index(begin = var_626_begin_0, end = var_626_end_0, end_mask = var_626_end_mask_0, x = window_17)[name = tensor("op_626")]; tensor rolled_15_interleave_0 = const()[name = tensor("rolled_15_interleave_0"), val = tensor(false)]; - tensor rolled_15 = concat(axis = var_33, interleave = rolled_15_interleave_0, values = (var_949, var_946))[name = tensor("rolled_15")]; + tensor rolled_15 = concat(axis = var_33, interleave = rolled_15_interleave_0, values = (var_626, var_623))[name = tensor("rolled_15")]; tensor clip_14 = clip(alpha = var_21, beta = var_20, x = clip_13)[name = tensor("clip_14")]; - tensor var_954 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_954")]; - tensor var_955_dtype_0 = const()[name = tensor("op_955_dtype_0"), val = tensor("fp32")]; - tensor var_956 = const()[name = tensor("op_956"), val = tensor([1, 16, 1])]; - tensor var_955 = cast(dtype = var_955_dtype_0, x = var_954)[name = tensor("cast_161")]; - tensor fill_mask_15 = reshape(shape = var_956, x = var_955)[name = tensor("fill_mask_15")]; + tensor var_631 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_631")]; + tensor var_632_dtype_0 = const()[name = tensor("op_632_dtype_0"), val = tensor("fp32")]; + tensor var_633 = const()[name = tensor("op_633"), val = tensor([1, 16, 1])]; + tensor var_632 = cast(dtype = var_632_dtype_0, x = var_631)[name = tensor("cast_161")]; + tensor fill_mask_15 = reshape(shape = var_633, x = var_632)[name = tensor("fill_mask_15")]; tensor x_expanded_15_reps_0 = const()[name = tensor("x_expanded_15_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_946)[name = tensor("x_expanded_15")]; - tensor var_960 = sub(x = var_17, y = fill_mask_15)[name = tensor("op_960")]; - tensor var_961 = mul(x = input_111, y = var_960)[name = tensor("op_961")]; - tensor var_962 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_962")]; - tensor filled_window_15 = add(x = var_961, y = var_962)[name = tensor("filled_window_15")]; - tensor var_964 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_964")]; - tensor var_965_dtype_0 = const()[name = tensor("op_965_dtype_0"), val = tensor("fp32")]; - tensor var_966 = const()[name = tensor("op_966"), val = tensor([1, 1, 1])]; - tensor var_965 = cast(dtype = var_965_dtype_0, x = var_964)[name = tensor("cast_160")]; - tensor use_roll_15 = reshape(shape = var_966, x = var_965)[name = tensor("use_roll_15")]; - tensor var_968 = sub(x = var_17, y = use_roll_15)[name = tensor("op_968")]; - tensor var_969 = mul(x = filled_window_15, y = var_968)[name = tensor("op_969")]; - tensor var_970 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_970")]; - tensor input_121 = add(x = var_969, y = var_970)[name = tensor("input_121")]; - tensor var_972 = const()[name = tensor("op_972"), val = tensor(0x1p+0)]; - tensor var_973 = add(x = clip_13, y = var_972)[name = tensor("op_973")]; - tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_973)[name = tensor("clip_15")]; - tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; - tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_121)[name = tensor("x_23")]; - tensor input_123_perm_0 = const()[name = tensor("input_123_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_49_pad_type_0 = const()[name = tensor("inputs_49_pad_type_0"), val = tensor("valid")]; - tensor inputs_49_strides_0 = const()[name = tensor("inputs_49_strides_0"), val = tensor([1])]; - tensor inputs_49_pad_0 = const()[name = tensor("inputs_49_pad_0"), val = tensor([0, 0])]; - tensor inputs_49_dilations_0 = const()[name = tensor("inputs_49_dilations_0"), val = tensor([1])]; - tensor inputs_49_groups_0 = const()[name = tensor("inputs_49_groups_0"), val = tensor(1)]; - tensor input_123 = transpose(perm = input_123_perm_0, x = x_23)[name = tensor("transpose_56")]; - tensor inputs_49 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_49_dilations_0, groups = inputs_49_groups_0, pad = inputs_49_pad_0, pad_type = inputs_49_pad_type_0, strides = inputs_49_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_123)[name = tensor("inputs_49")]; - tensor var_994_split_sizes_0 = const()[name = tensor("op_994_split_sizes_0"), val = tensor([256, 256])]; - tensor var_994_axis_0 = const()[name = tensor("op_994_axis_0"), val = tensor(1)]; - tensor var_994_0, tensor var_994_1 = split(axis = var_994_axis_0, split_sizes = var_994_split_sizes_0, x = inputs_49)[name = tensor("op_994")]; - tensor var_996 = sigmoid(x = var_994_1)[name = tensor("op_996")]; - tensor inputs_51 = mul(x = var_994_0, y = var_996)[name = tensor("inputs_51")]; - tensor outputs_aug_15_pad_type_0 = const()[name = tensor("outputs_aug_15_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_15_pad_0 = const()[name = tensor("outputs_aug_15_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_15_groups_0 = const()[name = tensor("outputs_aug_15_groups_0"), val = tensor(256)]; - tensor outputs_aug_15_strides_0 = const()[name = tensor("outputs_aug_15_strides_0"), val = tensor([1])]; - tensor outputs_aug_15_dilations_0 = const()[name = tensor("outputs_aug_15_dilations_0"), val = tensor([1])]; - tensor outputs_aug_15 = conv(dilations = outputs_aug_15_dilations_0, groups = outputs_aug_15_groups_0, pad = outputs_aug_15_pad_0, pad_type = outputs_aug_15_pad_type_0, strides = outputs_aug_15_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_51)[name = tensor("outputs_aug_15")]; - tensor input_125_begin_0 = const()[name = tensor("input_125_begin_0"), val = tensor([0, 0, 0])]; - tensor input_125_end_0 = const()[name = tensor("input_125_end_0"), val = tensor([1, 256, 16])]; - tensor input_125_end_mask_0 = const()[name = tensor("input_125_end_mask_0"), val = tensor([true, true, false])]; - tensor input_125 = slice_by_index(begin = input_125_begin_0, end = input_125_end_0, end_mask = input_125_end_mask_0, x = outputs_aug_15)[name = tensor("input_125")]; - tensor inputs_53 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_125)[name = tensor("inputs_53")]; - tensor input_127 = silu(x = inputs_53)[name = tensor("input_127")]; - tensor input_129_pad_type_0 = const()[name = tensor("input_129_pad_type_0"), val = tensor("valid")]; - tensor input_129_strides_0 = const()[name = tensor("input_129_strides_0"), val = tensor([1])]; - tensor input_129_pad_0 = const()[name = tensor("input_129_pad_0"), val = tensor([0, 0])]; - tensor input_129_dilations_0 = const()[name = tensor("input_129_dilations_0"), val = tensor([1])]; - tensor input_129_groups_0 = const()[name = tensor("input_129_groups_0"), val = tensor(1)]; - tensor input_129 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_129_dilations_0, groups = input_129_groups_0, pad = input_129_pad_0, pad_type = input_129_pad_type_0, strides = input_129_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_127)[name = tensor("input_129")]; - tensor conv_out_15_perm_0 = const()[name = tensor("conv_out_15_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1027_begin_0 = const()[name = tensor("op_1027_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1027_end_0 = const()[name = tensor("op_1027_end_0"), val = tensor([1, 16, 256])]; - tensor var_1027_end_mask_0 = const()[name = tensor("op_1027_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_15 = transpose(perm = conv_out_15_perm_0, x = input_129)[name = tensor("transpose_55")]; - tensor var_1027 = slice_by_index(begin = var_1027_begin_0, end = var_1027_end_0, end_mask = var_1027_end_mask_0, x = conv_out_15)[name = tensor("op_1027")]; - tensor var_1030_interleave_0 = const()[name = tensor("op_1030_interleave_0"), val = tensor(false)]; - tensor var_1030 = concat(axis = var_33, interleave = var_1030_interleave_0, values = (var_775, var_859, var_943, var_1027))[name = tensor("op_1030")]; - tensor input_131 = add(x = x_15, y = var_1030)[name = tensor("input_131")]; - tensor input_133_axes_0 = const()[name = tensor("input_133_axes_0"), val = tensor([-1])]; - tensor input_133 = layer_norm(axes = input_133_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_131)[name = tensor("input_133")]; - tensor inputs_55 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_133)[name = tensor("linear_17")]; - tensor input_135 = silu(x = inputs_55)[name = tensor("input_135")]; - tensor input_139 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_135)[name = tensor("linear_18")]; - tensor var_1053 = const()[name = tensor("op_1053"), val = tensor(0x1p-1)]; - tensor var_1054 = mul(x = input_139, y = var_1053)[name = tensor("op_1054")]; - tensor input_141 = add(x = var_1054, y = input_131)[name = tensor("input_141")]; - tensor input_143_axes_0 = const()[name = tensor("input_143_axes_0"), val = tensor([-1])]; - tensor input_143 = layer_norm(axes = input_143_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_141)[name = tensor("input_143")]; - tensor input_145_axes_0 = const()[name = tensor("input_145_axes_0"), val = tensor([-1])]; - tensor input_145 = layer_norm(axes = input_145_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_143)[name = tensor("input_145")]; - tensor inputs_57 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_145)[name = tensor("linear_19")]; - tensor input_147 = silu(x = inputs_57)[name = tensor("input_147")]; - tensor input_151 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_147)[name = tensor("linear_20")]; - tensor var_1083 = const()[name = tensor("op_1083"), val = tensor(0x1p-1)]; - tensor var_1084 = mul(x = input_151, y = var_1083)[name = tensor("op_1084")]; - tensor input_153 = add(x = var_1084, y = input_143)[name = tensor("input_153")]; - tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; - tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_153)[name = tensor("x_25")]; + tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_623)[name = tensor("x_expanded_15")]; + tensor var_637 = sub(x = var_17, y = fill_mask_15)[name = tensor("op_637")]; + tensor var_638 = mul(x = window_17, y = var_637)[name = tensor("op_638")]; + tensor var_639 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_639")]; + tensor filled_window_15 = add(x = var_638, y = var_639)[name = tensor("filled_window_15")]; + tensor var_641 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_641")]; + tensor var_642_dtype_0 = const()[name = tensor("op_642_dtype_0"), val = tensor("fp32")]; + tensor var_643 = const()[name = tensor("op_643"), val = tensor([1, 1, 1])]; + tensor var_642 = cast(dtype = var_642_dtype_0, x = var_641)[name = tensor("cast_160")]; + tensor use_roll_15 = reshape(shape = var_643, x = var_642)[name = tensor("use_roll_15")]; + tensor var_645 = sub(x = var_17, y = use_roll_15)[name = tensor("op_645")]; + tensor var_646 = mul(x = filled_window_15, y = var_645)[name = tensor("op_646")]; + tensor var_647 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_647")]; + tensor window_19 = add(x = var_646, y = var_647)[name = tensor("window_19")]; + tensor var_649 = const()[name = tensor("op_649"), val = tensor(0x1p+0)]; + tensor var_650 = add(x = clip_13, y = var_649)[name = tensor("op_650")]; + tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_650)[name = tensor("clip_15")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_30, interleave = input_61_interleave_0, values = (window_13, window_15, window_17, window_19))[name = tensor("input_61")]; + tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; + tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; + tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_13_pad_type_0 = const()[name = tensor("inputs_13_pad_type_0"), val = tensor("valid")]; + tensor inputs_13_strides_0 = const()[name = tensor("inputs_13_strides_0"), val = tensor([1])]; + tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; + tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; + tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; + tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; + tensor var_673_split_sizes_0 = const()[name = tensor("op_673_split_sizes_0"), val = tensor([256, 256])]; + tensor var_673_axis_0 = const()[name = tensor("op_673_axis_0"), val = tensor(1)]; + tensor var_673_0, tensor var_673_1 = split(axis = var_673_axis_0, split_sizes = var_673_split_sizes_0, x = inputs_13)[name = tensor("op_673")]; + tensor var_675 = sigmoid(x = var_673_1)[name = tensor("op_675")]; + tensor inputs_15 = mul(x = var_673_0, y = var_675)[name = tensor("inputs_15")]; + tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; + tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; + tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; + tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_15)[name = tensor("outputs_aug_3")]; + tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; + tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([4, 256, 16])]; + tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; + tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_3)[name = tensor("input_65")]; + tensor inputs_17 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_65)[name = tensor("inputs_17")]; + tensor input_67 = silu(x = inputs_17)[name = tensor("input_67")]; + tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; + tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; + tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; + tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; + tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; + tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; + tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; + tensor var_706_begin_0 = const()[name = tensor("op_706_begin_0"), val = tensor([0, -1, 0])]; + tensor var_706_end_0 = const()[name = tensor("op_706_end_0"), val = tensor([4, 16, 256])]; + tensor var_706_end_mask_0 = const()[name = tensor("op_706_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_706 = slice_by_index(begin = var_706_begin_0, end = var_706_end_0, end_mask = var_706_end_mask_0, x = conv_out_3)[name = tensor("op_706")]; + tensor var_708_perm_0 = const()[name = tensor("op_708_perm_0"), val = tensor([1, 0, 2])]; + tensor var_708 = transpose(perm = var_708_perm_0, x = var_706)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_708)[name = tensor("input_71")]; + tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; + tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; + tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; + tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; + tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; + tensor var_731 = const()[name = tensor("op_731"), val = tensor(0x1p-1)]; + tensor var_732 = mul(x = input_79, y = var_731)[name = tensor("op_732")]; + tensor input_81 = add(x = var_732, y = input_71)[name = tensor("input_81")]; + tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; + tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; + tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; + tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; + tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; + tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; + tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; + tensor var_761 = const()[name = tensor("op_761"), val = tensor(0x1p-1)]; + tensor var_762 = mul(x = input_91, y = var_761)[name = tensor("op_762")]; + tensor input_93 = add(x = var_762, y = input_83)[name = tensor("input_93")]; + tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; + tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; tensor prev_kv_5_end_0 = const()[name = tensor("prev_kv_5_end_0"), val = tensor([3, 1, 4, 64, 64])]; tensor prev_kv_5_end_mask_0 = const()[name = tensor("prev_kv_5_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1029,389 +800,274 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_1098 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_25)[name = tensor("linear_21")]; - tensor var_1099 = const()[name = tensor("op_1099"), val = tensor([1, 4, 4, 64])]; - tensor var_1100 = reshape(shape = var_1099, x = var_1098)[name = tensor("op_1100")]; + tensor var_776 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_777 = const()[name = tensor("op_777"), val = tensor([1, 4, 4, 64])]; + tensor var_778 = reshape(shape = var_777, x = var_776)[name = tensor("op_778")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1104 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_25)[name = tensor("linear_22")]; - tensor var_1105 = const()[name = tensor("op_1105"), val = tensor(0x1p-3)]; - tensor var_1106 = mul(x = var_1104, y = var_1105)[name = tensor("op_1106")]; - tensor var_1107 = const()[name = tensor("op_1107"), val = tensor([1, 4, 4, 64])]; - tensor var_1108 = reshape(shape = var_1107, x = var_1106)[name = tensor("op_1108")]; + tensor var_782 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_783 = const()[name = tensor("op_783"), val = tensor(0x1p-3)]; + tensor var_784 = mul(x = var_782, y = var_783)[name = tensor("op_784")]; + tensor var_785 = const()[name = tensor("op_785"), val = tensor([1, 4, 4, 64])]; + tensor var_786 = reshape(shape = var_785, x = var_784)[name = tensor("op_786")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1112 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_25)[name = tensor("linear_23")]; - tensor var_1113 = const()[name = tensor("op_1113"), val = tensor([1, 4, 4, 64])]; - tensor var_1114 = reshape(shape = var_1113, x = var_1112)[name = tensor("op_1114")]; + tensor var_790 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_791 = const()[name = tensor("op_791"), val = tensor([1, 4, 4, 64])]; + tensor var_792 = reshape(shape = var_791, x = var_790)[name = tensor("op_792")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_157 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_25)[name = tensor("linear_24")]; + tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; tensor s_t_5 = add(x = prev_scale_5, y = encoder__t_index)[name = tensor("s_t_5")]; tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_1108)[name = tensor("transpose_53")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_1100)[name = tensor("transpose_54")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_786)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_778)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_1124 = const()[name = tensor("op_1124"), val = tensor([4, 1])]; - tensor var_1125 = reshape(shape = var_1124, x = sqrt_s_t_5)[name = tensor("op_1125")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_1125)[name = tensor("M_5")]; - tensor var_1127 = mul(x = qk_5, y = M_5)[name = tensor("op_1127")]; + tensor var_802 = const()[name = tensor("op_802"), val = tensor([4, 1])]; + tensor var_803 = reshape(shape = var_802, x = sqrt_s_t_5)[name = tensor("op_803")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_803)[name = tensor("M_5")]; + tensor var_805 = mul(x = qk_5, y = M_5)[name = tensor("op_805")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_1114)[name = tensor("transpose_52")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_1127, y = v_5)[name = tensor("inner_5")]; - tensor var_1129_transpose_x_0 = const()[name = tensor("op_1129_transpose_x_0"), val = tensor(false)]; - tensor var_1129_transpose_y_0 = const()[name = tensor("op_1129_transpose_y_0"), val = tensor(false)]; - tensor var_1129 = matmul(transpose_x = var_1129_transpose_x_0, transpose_y = var_1129_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_1129")]; - tensor var_1130 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_1130")]; - tensor var_1131 = const()[name = tensor("op_1131"), val = tensor([1, 1, 4, 1])]; - tensor var_1132 = reshape(shape = var_1131, x = var_1130)[name = tensor("op_1132")]; - tensor cross_5 = mul(x = var_1129, y = var_1132)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_792)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_805, y = v_5)[name = tensor("inner_5")]; + tensor var_807_transpose_x_0 = const()[name = tensor("op_807_transpose_x_0"), val = tensor(false)]; + tensor var_807_transpose_y_0 = const()[name = tensor("op_807_transpose_y_0"), val = tensor(false)]; + tensor var_807 = matmul(transpose_x = var_807_transpose_x_0, transpose_y = var_807_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_807")]; + tensor var_808 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_808")]; + tensor var_809 = const()[name = tensor("op_809"), val = tensor([1, 1, 4, 1])]; + tensor var_810 = reshape(shape = var_809, x = var_808)[name = tensor("op_810")]; + tensor cross_5 = mul(x = var_807, y = var_810)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_1135 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_1135")]; - tensor var_1137_transpose_x_1 = const()[name = tensor("op_1137_transpose_x_1"), val = tensor(true)]; - tensor var_1137_transpose_y_1 = const()[name = tensor("op_1137_transpose_y_1"), val = tensor(false)]; - tensor var_1137 = matmul(transpose_x = var_1137_transpose_x_1, transpose_y = var_1137_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_1137")]; - tensor new_kv_unnorm_5 = add(x = var_1135, y = var_1137)[name = tensor("new_kv_unnorm_5")]; - tensor var_1139 = const()[name = tensor("op_1139"), val = tensor(0x1p+2)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_1139)[name = tensor("new_scale_5")]; - tensor var_1141 = sqrt(x = new_scale_5)[name = tensor("op_1141")]; - tensor var_1142 = real_div(x = new_kv_unnorm_5, y = var_1141)[name = tensor("op_1142")]; - tensor var_1143_perm_0 = const()[name = tensor("op_1143_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_813 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_813")]; + tensor var_815_transpose_x_1 = const()[name = tensor("op_815_transpose_x_1"), val = tensor(true)]; + tensor var_815_transpose_y_1 = const()[name = tensor("op_815_transpose_y_1"), val = tensor(false)]; + tensor var_815 = matmul(transpose_x = var_815_transpose_x_1, transpose_y = var_815_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_815")]; + tensor new_kv_unnorm_5 = add(x = var_813, y = var_815)[name = tensor("new_kv_unnorm_5")]; + tensor var_817 = const()[name = tensor("op_817"), val = tensor(0x1p+2)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_817)[name = tensor("new_scale_5")]; + tensor var_819 = sqrt(x = new_scale_5)[name = tensor("op_819")]; + tensor var_820 = real_div(x = new_kv_unnorm_5, y = var_819)[name = tensor("op_820")]; + tensor var_821_perm_0 = const()[name = tensor("op_821_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_1143 = transpose(perm = var_1143_perm_0, x = out_13)[name = tensor("transpose_51")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_1143)[name = tensor("out_15")]; - tensor var_1147 = const()[name = tensor("op_1147"), val = tensor([1, 4, 256])]; - tensor out_17 = reshape(shape = var_1147, x = out_15)[name = tensor("out_17")]; - tensor var_1149 = silu(x = input_157)[name = tensor("op_1149")]; - tensor input_159 = mul(x = var_1149, y = out_17)[name = tensor("input_159")]; - tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_159)[name = tensor("linear_25")]; - tensor x_27 = add(x = input_153, y = ret_out_5)[name = tensor("x_27")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor var_821 = transpose(perm = var_821_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_821)[name = tensor("out_15")]; + tensor var_825 = const()[name = tensor("op_825"), val = tensor([1, 4, 256])]; + tensor out_17 = reshape(shape = var_825, x = out_15)[name = tensor("out_17")]; + tensor var_827 = silu(x = input_97)[name = tensor("op_827")]; + tensor input_99 = mul(x = var_827, y = out_17)[name = tensor("input_99")]; + tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; + tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; + tensor window_21_begin_0 = const()[name = tensor("window_21_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_21_end_0 = const()[name = tensor("window_21_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_21_end_mask_0 = const()[name = tensor("window_21_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_21_squeeze_mask_0 = const()[name = tensor("window_21_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_21 = slice_by_index(begin = window_21_begin_0, end = window_21_end_0, end_mask = window_21_end_mask_0, squeeze_mask = window_21_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_21")]; tensor filled_17_begin_0 = const()[name = tensor("filled_17_begin_0"), val = tensor([2, 0])]; tensor filled_17_end_0 = const()[name = tensor("filled_17_end_0"), val = tensor([3, 1])]; tensor filled_17_end_mask_0 = const()[name = tensor("filled_17_end_mask_0"), val = tensor([false, true])]; tensor filled_17_squeeze_mask_0 = const()[name = tensor("filled_17_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_17 = slice_by_index(begin = filled_17_begin_0, end = filled_17_end_0, end_mask = filled_17_end_mask_0, squeeze_mask = filled_17_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_17")]; - tensor var_1158_begin_0 = const()[name = tensor("op_1158_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1158_end_0 = const()[name = tensor("op_1158_end_0"), val = tensor([1, 1, 256])]; - tensor var_1158_end_mask_0 = const()[name = tensor("op_1158_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1158 = slice_by_index(begin = var_1158_begin_0, end = var_1158_end_0, end_mask = var_1158_end_mask_0, x = x_27)[name = tensor("op_1158")]; - tensor var_1161_begin_0 = const()[name = tensor("op_1161_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1161_end_0 = const()[name = tensor("op_1161_end_0"), val = tensor([1, 16, 256])]; - tensor var_1161_end_mask_0 = const()[name = tensor("op_1161_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1161 = slice_by_index(begin = var_1161_begin_0, end = var_1161_end_0, end_mask = var_1161_end_mask_0, x = window_5)[name = tensor("op_1161")]; + tensor var_836_begin_0 = const()[name = tensor("op_836_begin_0"), val = tensor([0, 0, 0])]; + tensor var_836_end_0 = const()[name = tensor("op_836_end_0"), val = tensor([1, 1, 256])]; + tensor var_836_end_mask_0 = const()[name = tensor("op_836_end_mask_0"), val = tensor([true, false, true])]; + tensor var_836 = slice_by_index(begin = var_836_begin_0, end = var_836_end_0, end_mask = var_836_end_mask_0, x = x_15)[name = tensor("op_836")]; + tensor var_839_begin_0 = const()[name = tensor("op_839_begin_0"), val = tensor([0, 1, 0])]; + tensor var_839_end_0 = const()[name = tensor("op_839_end_0"), val = tensor([1, 16, 256])]; + tensor var_839_end_mask_0 = const()[name = tensor("op_839_end_mask_0"), val = tensor([true, true, true])]; + tensor var_839 = slice_by_index(begin = var_839_begin_0, end = var_839_end_0, end_mask = var_839_end_mask_0, x = window_21)[name = tensor("op_839")]; tensor rolled_17_interleave_0 = const()[name = tensor("rolled_17_interleave_0"), val = tensor(false)]; - tensor rolled_17 = concat(axis = var_33, interleave = rolled_17_interleave_0, values = (var_1161, var_1158))[name = tensor("rolled_17")]; + tensor rolled_17 = concat(axis = var_33, interleave = rolled_17_interleave_0, values = (var_839, var_836))[name = tensor("rolled_17")]; tensor clip_16 = clip(alpha = var_21, beta = var_20, x = filled_17)[name = tensor("clip_16")]; - tensor var_1166 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_1166")]; - tensor var_1167_dtype_0 = const()[name = tensor("op_1167_dtype_0"), val = tensor("fp32")]; - tensor var_1168 = const()[name = tensor("op_1168"), val = tensor([1, 16, 1])]; - tensor var_1167 = cast(dtype = var_1167_dtype_0, x = var_1166)[name = tensor("cast_159")]; - tensor fill_mask_17 = reshape(shape = var_1168, x = var_1167)[name = tensor("fill_mask_17")]; + tensor var_844 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_844")]; + tensor var_845_dtype_0 = const()[name = tensor("op_845_dtype_0"), val = tensor("fp32")]; + tensor var_846 = const()[name = tensor("op_846"), val = tensor([1, 16, 1])]; + tensor var_845 = cast(dtype = var_845_dtype_0, x = var_844)[name = tensor("cast_159")]; + tensor fill_mask_17 = reshape(shape = var_846, x = var_845)[name = tensor("fill_mask_17")]; tensor x_expanded_17_reps_0 = const()[name = tensor("x_expanded_17_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_1158)[name = tensor("x_expanded_17")]; - tensor var_1172 = sub(x = var_17, y = fill_mask_17)[name = tensor("op_1172")]; - tensor var_1173 = mul(x = window_5, y = var_1172)[name = tensor("op_1173")]; - tensor var_1174 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_1174")]; - tensor filled_window_17 = add(x = var_1173, y = var_1174)[name = tensor("filled_window_17")]; - tensor var_1176 = greater_equal(x = filled_17, y = var_16)[name = tensor("op_1176")]; - tensor var_1177_dtype_0 = const()[name = tensor("op_1177_dtype_0"), val = tensor("fp32")]; - tensor var_1178 = const()[name = tensor("op_1178"), val = tensor([1, 1, 1])]; - tensor var_1177 = cast(dtype = var_1177_dtype_0, x = var_1176)[name = tensor("cast_158")]; - tensor use_roll_17 = reshape(shape = var_1178, x = var_1177)[name = tensor("use_roll_17")]; - tensor var_1180 = sub(x = var_17, y = use_roll_17)[name = tensor("op_1180")]; - tensor var_1181 = mul(x = filled_window_17, y = var_1180)[name = tensor("op_1181")]; - tensor var_1182 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_1182")]; - tensor input_161 = add(x = var_1181, y = var_1182)[name = tensor("input_161")]; - tensor var_1184 = const()[name = tensor("op_1184"), val = tensor(0x1p+0)]; - tensor var_1185 = add(x = filled_17, y = var_1184)[name = tensor("op_1185")]; - tensor clip_17 = clip(alpha = var_21, beta = var_16, x = var_1185)[name = tensor("clip_17")]; - tensor x_29_axes_0 = const()[name = tensor("x_29_axes_0"), val = tensor([-1])]; - tensor x_29 = layer_norm(axes = x_29_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_161)[name = tensor("x_29")]; - tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_59_pad_type_0 = const()[name = tensor("inputs_59_pad_type_0"), val = tensor("valid")]; - tensor inputs_59_strides_0 = const()[name = tensor("inputs_59_strides_0"), val = tensor([1])]; - tensor inputs_59_pad_0 = const()[name = tensor("inputs_59_pad_0"), val = tensor([0, 0])]; - tensor inputs_59_dilations_0 = const()[name = tensor("inputs_59_dilations_0"), val = tensor([1])]; - tensor inputs_59_groups_0 = const()[name = tensor("inputs_59_groups_0"), val = tensor(1)]; - tensor input_163 = transpose(perm = input_163_perm_0, x = x_29)[name = tensor("transpose_50")]; - tensor inputs_59 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_59_dilations_0, groups = inputs_59_groups_0, pad = inputs_59_pad_0, pad_type = inputs_59_pad_type_0, strides = inputs_59_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_163)[name = tensor("inputs_59")]; - tensor var_1206_split_sizes_0 = const()[name = tensor("op_1206_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1206_axis_0 = const()[name = tensor("op_1206_axis_0"), val = tensor(1)]; - tensor var_1206_0, tensor var_1206_1 = split(axis = var_1206_axis_0, split_sizes = var_1206_split_sizes_0, x = inputs_59)[name = tensor("op_1206")]; - tensor var_1208 = sigmoid(x = var_1206_1)[name = tensor("op_1208")]; - tensor inputs_61 = mul(x = var_1206_0, y = var_1208)[name = tensor("inputs_61")]; - tensor outputs_aug_17_pad_type_0 = const()[name = tensor("outputs_aug_17_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_17_pad_0 = const()[name = tensor("outputs_aug_17_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_17_groups_0 = const()[name = tensor("outputs_aug_17_groups_0"), val = tensor(256)]; - tensor outputs_aug_17_strides_0 = const()[name = tensor("outputs_aug_17_strides_0"), val = tensor([1])]; - tensor outputs_aug_17_dilations_0 = const()[name = tensor("outputs_aug_17_dilations_0"), val = tensor([1])]; - tensor outputs_aug_17 = conv(dilations = outputs_aug_17_dilations_0, groups = outputs_aug_17_groups_0, pad = outputs_aug_17_pad_0, pad_type = outputs_aug_17_pad_type_0, strides = outputs_aug_17_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_61)[name = tensor("outputs_aug_17")]; - tensor input_165_begin_0 = const()[name = tensor("input_165_begin_0"), val = tensor([0, 0, 0])]; - tensor input_165_end_0 = const()[name = tensor("input_165_end_0"), val = tensor([1, 256, 16])]; - tensor input_165_end_mask_0 = const()[name = tensor("input_165_end_mask_0"), val = tensor([true, true, false])]; - tensor input_165 = slice_by_index(begin = input_165_begin_0, end = input_165_end_0, end_mask = input_165_end_mask_0, x = outputs_aug_17)[name = tensor("input_165")]; - tensor inputs_63 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_165)[name = tensor("inputs_63")]; - tensor input_167 = silu(x = inputs_63)[name = tensor("input_167")]; - tensor input_169_pad_type_0 = const()[name = tensor("input_169_pad_type_0"), val = tensor("valid")]; - tensor input_169_strides_0 = const()[name = tensor("input_169_strides_0"), val = tensor([1])]; - tensor input_169_pad_0 = const()[name = tensor("input_169_pad_0"), val = tensor([0, 0])]; - tensor input_169_dilations_0 = const()[name = tensor("input_169_dilations_0"), val = tensor([1])]; - tensor input_169_groups_0 = const()[name = tensor("input_169_groups_0"), val = tensor(1)]; - tensor input_169 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_169_dilations_0, groups = input_169_groups_0, pad = input_169_pad_0, pad_type = input_169_pad_type_0, strides = input_169_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_167)[name = tensor("input_169")]; - tensor conv_out_17_perm_0 = const()[name = tensor("conv_out_17_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1239_begin_0 = const()[name = tensor("op_1239_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1239_end_0 = const()[name = tensor("op_1239_end_0"), val = tensor([1, 16, 256])]; - tensor var_1239_end_mask_0 = const()[name = tensor("op_1239_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_17 = transpose(perm = conv_out_17_perm_0, x = input_169)[name = tensor("transpose_49")]; - tensor var_1239 = slice_by_index(begin = var_1239_begin_0, end = var_1239_end_0, end_mask = var_1239_end_mask_0, x = conv_out_17)[name = tensor("op_1239")]; - tensor var_1242_begin_0 = const()[name = tensor("op_1242_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1242_end_0 = const()[name = tensor("op_1242_end_0"), val = tensor([1, 2, 256])]; - tensor var_1242_end_mask_0 = const()[name = tensor("op_1242_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1242 = slice_by_index(begin = var_1242_begin_0, end = var_1242_end_0, end_mask = var_1242_end_mask_0, x = x_27)[name = tensor("op_1242")]; - tensor var_1245_begin_0 = const()[name = tensor("op_1245_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1245_end_0 = const()[name = tensor("op_1245_end_0"), val = tensor([1, 16, 256])]; - tensor var_1245_end_mask_0 = const()[name = tensor("op_1245_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1245 = slice_by_index(begin = var_1245_begin_0, end = var_1245_end_0, end_mask = var_1245_end_mask_0, x = input_161)[name = tensor("op_1245")]; + tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_836)[name = tensor("x_expanded_17")]; + tensor var_850 = sub(x = var_17, y = fill_mask_17)[name = tensor("op_850")]; + tensor var_851 = mul(x = window_21, y = var_850)[name = tensor("op_851")]; + tensor var_852 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_852")]; + tensor filled_window_17 = add(x = var_851, y = var_852)[name = tensor("filled_window_17")]; + tensor var_854 = greater_equal(x = filled_17, y = var_16)[name = tensor("op_854")]; + tensor var_855_dtype_0 = const()[name = tensor("op_855_dtype_0"), val = tensor("fp32")]; + tensor var_856 = const()[name = tensor("op_856"), val = tensor([1, 1, 1])]; + tensor var_855 = cast(dtype = var_855_dtype_0, x = var_854)[name = tensor("cast_158")]; + tensor use_roll_17 = reshape(shape = var_856, x = var_855)[name = tensor("use_roll_17")]; + tensor var_858 = sub(x = var_17, y = use_roll_17)[name = tensor("op_858")]; + tensor var_859 = mul(x = filled_window_17, y = var_858)[name = tensor("op_859")]; + tensor var_860 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_860")]; + tensor window_23 = add(x = var_859, y = var_860)[name = tensor("window_23")]; + tensor var_862 = const()[name = tensor("op_862"), val = tensor(0x1p+0)]; + tensor var_863 = add(x = filled_17, y = var_862)[name = tensor("op_863")]; + tensor clip_17 = clip(alpha = var_21, beta = var_16, x = var_863)[name = tensor("clip_17")]; + tensor var_866_begin_0 = const()[name = tensor("op_866_begin_0"), val = tensor([0, 1, 0])]; + tensor var_866_end_0 = const()[name = tensor("op_866_end_0"), val = tensor([1, 2, 256])]; + tensor var_866_end_mask_0 = const()[name = tensor("op_866_end_mask_0"), val = tensor([true, false, true])]; + tensor var_866 = slice_by_index(begin = var_866_begin_0, end = var_866_end_0, end_mask = var_866_end_mask_0, x = x_15)[name = tensor("op_866")]; + tensor var_869_begin_0 = const()[name = tensor("op_869_begin_0"), val = tensor([0, 1, 0])]; + tensor var_869_end_0 = const()[name = tensor("op_869_end_0"), val = tensor([1, 16, 256])]; + tensor var_869_end_mask_0 = const()[name = tensor("op_869_end_mask_0"), val = tensor([true, true, true])]; + tensor var_869 = slice_by_index(begin = var_869_begin_0, end = var_869_end_0, end_mask = var_869_end_mask_0, x = window_23)[name = tensor("op_869")]; tensor rolled_19_interleave_0 = const()[name = tensor("rolled_19_interleave_0"), val = tensor(false)]; - tensor rolled_19 = concat(axis = var_33, interleave = rolled_19_interleave_0, values = (var_1245, var_1242))[name = tensor("rolled_19")]; + tensor rolled_19 = concat(axis = var_33, interleave = rolled_19_interleave_0, values = (var_869, var_866))[name = tensor("rolled_19")]; tensor clip_18 = clip(alpha = var_21, beta = var_20, x = clip_17)[name = tensor("clip_18")]; - tensor var_1250 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_1250")]; - tensor var_1251_dtype_0 = const()[name = tensor("op_1251_dtype_0"), val = tensor("fp32")]; - tensor var_1252 = const()[name = tensor("op_1252"), val = tensor([1, 16, 1])]; - tensor var_1251 = cast(dtype = var_1251_dtype_0, x = var_1250)[name = tensor("cast_157")]; - tensor fill_mask_19 = reshape(shape = var_1252, x = var_1251)[name = tensor("fill_mask_19")]; + tensor var_874 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_874")]; + tensor var_875_dtype_0 = const()[name = tensor("op_875_dtype_0"), val = tensor("fp32")]; + tensor var_876 = const()[name = tensor("op_876"), val = tensor([1, 16, 1])]; + tensor var_875 = cast(dtype = var_875_dtype_0, x = var_874)[name = tensor("cast_157")]; + tensor fill_mask_19 = reshape(shape = var_876, x = var_875)[name = tensor("fill_mask_19")]; tensor x_expanded_19_reps_0 = const()[name = tensor("x_expanded_19_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_1242)[name = tensor("x_expanded_19")]; - tensor var_1256 = sub(x = var_17, y = fill_mask_19)[name = tensor("op_1256")]; - tensor var_1257 = mul(x = input_161, y = var_1256)[name = tensor("op_1257")]; - tensor var_1258 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_1258")]; - tensor filled_window_19 = add(x = var_1257, y = var_1258)[name = tensor("filled_window_19")]; - tensor var_1260 = greater_equal(x = clip_17, y = var_16)[name = tensor("op_1260")]; - tensor var_1261_dtype_0 = const()[name = tensor("op_1261_dtype_0"), val = tensor("fp32")]; - tensor var_1262 = const()[name = tensor("op_1262"), val = tensor([1, 1, 1])]; - tensor var_1261 = cast(dtype = var_1261_dtype_0, x = var_1260)[name = tensor("cast_156")]; - tensor use_roll_19 = reshape(shape = var_1262, x = var_1261)[name = tensor("use_roll_19")]; - tensor var_1264 = sub(x = var_17, y = use_roll_19)[name = tensor("op_1264")]; - tensor var_1265 = mul(x = filled_window_19, y = var_1264)[name = tensor("op_1265")]; - tensor var_1266 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_1266")]; - tensor input_171 = add(x = var_1265, y = var_1266)[name = tensor("input_171")]; - tensor var_1268 = const()[name = tensor("op_1268"), val = tensor(0x1p+0)]; - tensor var_1269 = add(x = clip_17, y = var_1268)[name = tensor("op_1269")]; - tensor clip_19 = clip(alpha = var_21, beta = var_16, x = var_1269)[name = tensor("clip_19")]; - tensor x_31_axes_0 = const()[name = tensor("x_31_axes_0"), val = tensor([-1])]; - tensor x_31 = layer_norm(axes = x_31_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_171)[name = tensor("x_31")]; - tensor input_173_perm_0 = const()[name = tensor("input_173_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_65_pad_type_0 = const()[name = tensor("inputs_65_pad_type_0"), val = tensor("valid")]; - tensor inputs_65_strides_0 = const()[name = tensor("inputs_65_strides_0"), val = tensor([1])]; - tensor inputs_65_pad_0 = const()[name = tensor("inputs_65_pad_0"), val = tensor([0, 0])]; - tensor inputs_65_dilations_0 = const()[name = tensor("inputs_65_dilations_0"), val = tensor([1])]; - tensor inputs_65_groups_0 = const()[name = tensor("inputs_65_groups_0"), val = tensor(1)]; - tensor input_173 = transpose(perm = input_173_perm_0, x = x_31)[name = tensor("transpose_48")]; - tensor inputs_65 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_65_dilations_0, groups = inputs_65_groups_0, pad = inputs_65_pad_0, pad_type = inputs_65_pad_type_0, strides = inputs_65_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_173)[name = tensor("inputs_65")]; - tensor var_1290_split_sizes_0 = const()[name = tensor("op_1290_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1290_axis_0 = const()[name = tensor("op_1290_axis_0"), val = tensor(1)]; - tensor var_1290_0, tensor var_1290_1 = split(axis = var_1290_axis_0, split_sizes = var_1290_split_sizes_0, x = inputs_65)[name = tensor("op_1290")]; - tensor var_1292 = sigmoid(x = var_1290_1)[name = tensor("op_1292")]; - tensor inputs_67 = mul(x = var_1290_0, y = var_1292)[name = tensor("inputs_67")]; - tensor outputs_aug_19_pad_type_0 = const()[name = tensor("outputs_aug_19_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_19_pad_0 = const()[name = tensor("outputs_aug_19_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_19_groups_0 = const()[name = tensor("outputs_aug_19_groups_0"), val = tensor(256)]; - tensor outputs_aug_19_strides_0 = const()[name = tensor("outputs_aug_19_strides_0"), val = tensor([1])]; - tensor outputs_aug_19_dilations_0 = const()[name = tensor("outputs_aug_19_dilations_0"), val = tensor([1])]; - tensor outputs_aug_19 = conv(dilations = outputs_aug_19_dilations_0, groups = outputs_aug_19_groups_0, pad = outputs_aug_19_pad_0, pad_type = outputs_aug_19_pad_type_0, strides = outputs_aug_19_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_67)[name = tensor("outputs_aug_19")]; - tensor input_175_begin_0 = const()[name = tensor("input_175_begin_0"), val = tensor([0, 0, 0])]; - tensor input_175_end_0 = const()[name = tensor("input_175_end_0"), val = tensor([1, 256, 16])]; - tensor input_175_end_mask_0 = const()[name = tensor("input_175_end_mask_0"), val = tensor([true, true, false])]; - tensor input_175 = slice_by_index(begin = input_175_begin_0, end = input_175_end_0, end_mask = input_175_end_mask_0, x = outputs_aug_19)[name = tensor("input_175")]; - tensor inputs_69 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_175)[name = tensor("inputs_69")]; - tensor input_177 = silu(x = inputs_69)[name = tensor("input_177")]; - tensor input_179_pad_type_0 = const()[name = tensor("input_179_pad_type_0"), val = tensor("valid")]; - tensor input_179_strides_0 = const()[name = tensor("input_179_strides_0"), val = tensor([1])]; - tensor input_179_pad_0 = const()[name = tensor("input_179_pad_0"), val = tensor([0, 0])]; - tensor input_179_dilations_0 = const()[name = tensor("input_179_dilations_0"), val = tensor([1])]; - tensor input_179_groups_0 = const()[name = tensor("input_179_groups_0"), val = tensor(1)]; - tensor input_179 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_179_dilations_0, groups = input_179_groups_0, pad = input_179_pad_0, pad_type = input_179_pad_type_0, strides = input_179_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_177)[name = tensor("input_179")]; - tensor conv_out_19_perm_0 = const()[name = tensor("conv_out_19_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1323_begin_0 = const()[name = tensor("op_1323_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1323_end_0 = const()[name = tensor("op_1323_end_0"), val = tensor([1, 16, 256])]; - tensor var_1323_end_mask_0 = const()[name = tensor("op_1323_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_19 = transpose(perm = conv_out_19_perm_0, x = input_179)[name = tensor("transpose_47")]; - tensor var_1323 = slice_by_index(begin = var_1323_begin_0, end = var_1323_end_0, end_mask = var_1323_end_mask_0, x = conv_out_19)[name = tensor("op_1323")]; - tensor var_1326_begin_0 = const()[name = tensor("op_1326_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1326_end_0 = const()[name = tensor("op_1326_end_0"), val = tensor([1, 3, 256])]; - tensor var_1326_end_mask_0 = const()[name = tensor("op_1326_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1326 = slice_by_index(begin = var_1326_begin_0, end = var_1326_end_0, end_mask = var_1326_end_mask_0, x = x_27)[name = tensor("op_1326")]; - tensor var_1329_begin_0 = const()[name = tensor("op_1329_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1329_end_0 = const()[name = tensor("op_1329_end_0"), val = tensor([1, 16, 256])]; - tensor var_1329_end_mask_0 = const()[name = tensor("op_1329_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1329 = slice_by_index(begin = var_1329_begin_0, end = var_1329_end_0, end_mask = var_1329_end_mask_0, x = input_171)[name = tensor("op_1329")]; + tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_866)[name = tensor("x_expanded_19")]; + tensor var_880 = sub(x = var_17, y = fill_mask_19)[name = tensor("op_880")]; + tensor var_881 = mul(x = window_23, y = var_880)[name = tensor("op_881")]; + tensor var_882 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_882")]; + tensor filled_window_19 = add(x = var_881, y = var_882)[name = tensor("filled_window_19")]; + tensor var_884 = greater_equal(x = clip_17, y = var_16)[name = tensor("op_884")]; + tensor var_885_dtype_0 = const()[name = tensor("op_885_dtype_0"), val = tensor("fp32")]; + tensor var_886 = const()[name = tensor("op_886"), val = tensor([1, 1, 1])]; + tensor var_885 = cast(dtype = var_885_dtype_0, x = var_884)[name = tensor("cast_156")]; + tensor use_roll_19 = reshape(shape = var_886, x = var_885)[name = tensor("use_roll_19")]; + tensor var_888 = sub(x = var_17, y = use_roll_19)[name = tensor("op_888")]; + tensor var_889 = mul(x = filled_window_19, y = var_888)[name = tensor("op_889")]; + tensor var_890 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_890")]; + tensor window_25 = add(x = var_889, y = var_890)[name = tensor("window_25")]; + tensor var_892 = const()[name = tensor("op_892"), val = tensor(0x1p+0)]; + tensor var_893 = add(x = clip_17, y = var_892)[name = tensor("op_893")]; + tensor clip_19 = clip(alpha = var_21, beta = var_16, x = var_893)[name = tensor("clip_19")]; + tensor var_896_begin_0 = const()[name = tensor("op_896_begin_0"), val = tensor([0, 2, 0])]; + tensor var_896_end_0 = const()[name = tensor("op_896_end_0"), val = tensor([1, 3, 256])]; + tensor var_896_end_mask_0 = const()[name = tensor("op_896_end_mask_0"), val = tensor([true, false, true])]; + tensor var_896 = slice_by_index(begin = var_896_begin_0, end = var_896_end_0, end_mask = var_896_end_mask_0, x = x_15)[name = tensor("op_896")]; + tensor var_899_begin_0 = const()[name = tensor("op_899_begin_0"), val = tensor([0, 1, 0])]; + tensor var_899_end_0 = const()[name = tensor("op_899_end_0"), val = tensor([1, 16, 256])]; + tensor var_899_end_mask_0 = const()[name = tensor("op_899_end_mask_0"), val = tensor([true, true, true])]; + tensor var_899 = slice_by_index(begin = var_899_begin_0, end = var_899_end_0, end_mask = var_899_end_mask_0, x = window_25)[name = tensor("op_899")]; tensor rolled_21_interleave_0 = const()[name = tensor("rolled_21_interleave_0"), val = tensor(false)]; - tensor rolled_21 = concat(axis = var_33, interleave = rolled_21_interleave_0, values = (var_1329, var_1326))[name = tensor("rolled_21")]; + tensor rolled_21 = concat(axis = var_33, interleave = rolled_21_interleave_0, values = (var_899, var_896))[name = tensor("rolled_21")]; tensor clip_20 = clip(alpha = var_21, beta = var_20, x = clip_19)[name = tensor("clip_20")]; - tensor var_1334 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_1334")]; - tensor var_1335_dtype_0 = const()[name = tensor("op_1335_dtype_0"), val = tensor("fp32")]; - tensor var_1336 = const()[name = tensor("op_1336"), val = tensor([1, 16, 1])]; - tensor var_1335 = cast(dtype = var_1335_dtype_0, x = var_1334)[name = tensor("cast_155")]; - tensor fill_mask_21 = reshape(shape = var_1336, x = var_1335)[name = tensor("fill_mask_21")]; + tensor var_904 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_904")]; + tensor var_905_dtype_0 = const()[name = tensor("op_905_dtype_0"), val = tensor("fp32")]; + tensor var_906 = const()[name = tensor("op_906"), val = tensor([1, 16, 1])]; + tensor var_905 = cast(dtype = var_905_dtype_0, x = var_904)[name = tensor("cast_155")]; + tensor fill_mask_21 = reshape(shape = var_906, x = var_905)[name = tensor("fill_mask_21")]; tensor x_expanded_21_reps_0 = const()[name = tensor("x_expanded_21_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_1326)[name = tensor("x_expanded_21")]; - tensor var_1340 = sub(x = var_17, y = fill_mask_21)[name = tensor("op_1340")]; - tensor var_1341 = mul(x = input_171, y = var_1340)[name = tensor("op_1341")]; - tensor var_1342 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_1342")]; - tensor filled_window_21 = add(x = var_1341, y = var_1342)[name = tensor("filled_window_21")]; - tensor var_1344 = greater_equal(x = clip_19, y = var_16)[name = tensor("op_1344")]; - tensor var_1345_dtype_0 = const()[name = tensor("op_1345_dtype_0"), val = tensor("fp32")]; - tensor var_1346 = const()[name = tensor("op_1346"), val = tensor([1, 1, 1])]; - tensor var_1345 = cast(dtype = var_1345_dtype_0, x = var_1344)[name = tensor("cast_154")]; - tensor use_roll_21 = reshape(shape = var_1346, x = var_1345)[name = tensor("use_roll_21")]; - tensor var_1348 = sub(x = var_17, y = use_roll_21)[name = tensor("op_1348")]; - tensor var_1349 = mul(x = filled_window_21, y = var_1348)[name = tensor("op_1349")]; - tensor var_1350 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_1350")]; - tensor input_181 = add(x = var_1349, y = var_1350)[name = tensor("input_181")]; - tensor var_1352 = const()[name = tensor("op_1352"), val = tensor(0x1p+0)]; - tensor var_1353 = add(x = clip_19, y = var_1352)[name = tensor("op_1353")]; - tensor clip_21 = clip(alpha = var_21, beta = var_16, x = var_1353)[name = tensor("clip_21")]; - tensor x_33_axes_0 = const()[name = tensor("x_33_axes_0"), val = tensor([-1])]; - tensor x_33 = layer_norm(axes = x_33_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_181)[name = tensor("x_33")]; - tensor input_183_perm_0 = const()[name = tensor("input_183_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_71_pad_type_0 = const()[name = tensor("inputs_71_pad_type_0"), val = tensor("valid")]; - tensor inputs_71_strides_0 = const()[name = tensor("inputs_71_strides_0"), val = tensor([1])]; - tensor inputs_71_pad_0 = const()[name = tensor("inputs_71_pad_0"), val = tensor([0, 0])]; - tensor inputs_71_dilations_0 = const()[name = tensor("inputs_71_dilations_0"), val = tensor([1])]; - tensor inputs_71_groups_0 = const()[name = tensor("inputs_71_groups_0"), val = tensor(1)]; - tensor input_183 = transpose(perm = input_183_perm_0, x = x_33)[name = tensor("transpose_46")]; - tensor inputs_71 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_71_dilations_0, groups = inputs_71_groups_0, pad = inputs_71_pad_0, pad_type = inputs_71_pad_type_0, strides = inputs_71_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_183)[name = tensor("inputs_71")]; - tensor var_1374_split_sizes_0 = const()[name = tensor("op_1374_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1374_axis_0 = const()[name = tensor("op_1374_axis_0"), val = tensor(1)]; - tensor var_1374_0, tensor var_1374_1 = split(axis = var_1374_axis_0, split_sizes = var_1374_split_sizes_0, x = inputs_71)[name = tensor("op_1374")]; - tensor var_1376 = sigmoid(x = var_1374_1)[name = tensor("op_1376")]; - tensor inputs_73 = mul(x = var_1374_0, y = var_1376)[name = tensor("inputs_73")]; - tensor outputs_aug_21_pad_type_0 = const()[name = tensor("outputs_aug_21_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_21_pad_0 = const()[name = tensor("outputs_aug_21_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_21_groups_0 = const()[name = tensor("outputs_aug_21_groups_0"), val = tensor(256)]; - tensor outputs_aug_21_strides_0 = const()[name = tensor("outputs_aug_21_strides_0"), val = tensor([1])]; - tensor outputs_aug_21_dilations_0 = const()[name = tensor("outputs_aug_21_dilations_0"), val = tensor([1])]; - tensor outputs_aug_21 = conv(dilations = outputs_aug_21_dilations_0, groups = outputs_aug_21_groups_0, pad = outputs_aug_21_pad_0, pad_type = outputs_aug_21_pad_type_0, strides = outputs_aug_21_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_73)[name = tensor("outputs_aug_21")]; - tensor input_185_begin_0 = const()[name = tensor("input_185_begin_0"), val = tensor([0, 0, 0])]; - tensor input_185_end_0 = const()[name = tensor("input_185_end_0"), val = tensor([1, 256, 16])]; - tensor input_185_end_mask_0 = const()[name = tensor("input_185_end_mask_0"), val = tensor([true, true, false])]; - tensor input_185 = slice_by_index(begin = input_185_begin_0, end = input_185_end_0, end_mask = input_185_end_mask_0, x = outputs_aug_21)[name = tensor("input_185")]; - tensor inputs_75 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_185)[name = tensor("inputs_75")]; - tensor input_187 = silu(x = inputs_75)[name = tensor("input_187")]; - tensor input_189_pad_type_0 = const()[name = tensor("input_189_pad_type_0"), val = tensor("valid")]; - tensor input_189_strides_0 = const()[name = tensor("input_189_strides_0"), val = tensor([1])]; - tensor input_189_pad_0 = const()[name = tensor("input_189_pad_0"), val = tensor([0, 0])]; - tensor input_189_dilations_0 = const()[name = tensor("input_189_dilations_0"), val = tensor([1])]; - tensor input_189_groups_0 = const()[name = tensor("input_189_groups_0"), val = tensor(1)]; - tensor input_189 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_189_dilations_0, groups = input_189_groups_0, pad = input_189_pad_0, pad_type = input_189_pad_type_0, strides = input_189_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_187)[name = tensor("input_189")]; - tensor conv_out_21_perm_0 = const()[name = tensor("conv_out_21_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1407_begin_0 = const()[name = tensor("op_1407_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1407_end_0 = const()[name = tensor("op_1407_end_0"), val = tensor([1, 16, 256])]; - tensor var_1407_end_mask_0 = const()[name = tensor("op_1407_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_21 = transpose(perm = conv_out_21_perm_0, x = input_189)[name = tensor("transpose_45")]; - tensor var_1407 = slice_by_index(begin = var_1407_begin_0, end = var_1407_end_0, end_mask = var_1407_end_mask_0, x = conv_out_21)[name = tensor("op_1407")]; - tensor var_1410_begin_0 = const()[name = tensor("op_1410_begin_0"), val = tensor([0, 3, 0])]; - tensor var_1410_end_0 = const()[name = tensor("op_1410_end_0"), val = tensor([1, 1, 256])]; - tensor var_1410_end_mask_0 = const()[name = tensor("op_1410_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1410 = slice_by_index(begin = var_1410_begin_0, end = var_1410_end_0, end_mask = var_1410_end_mask_0, x = x_27)[name = tensor("op_1410")]; - tensor var_1413_begin_0 = const()[name = tensor("op_1413_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1413_end_0 = const()[name = tensor("op_1413_end_0"), val = tensor([1, 16, 256])]; - tensor var_1413_end_mask_0 = const()[name = tensor("op_1413_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1413 = slice_by_index(begin = var_1413_begin_0, end = var_1413_end_0, end_mask = var_1413_end_mask_0, x = input_181)[name = tensor("op_1413")]; + tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_896)[name = tensor("x_expanded_21")]; + tensor var_910 = sub(x = var_17, y = fill_mask_21)[name = tensor("op_910")]; + tensor var_911 = mul(x = window_25, y = var_910)[name = tensor("op_911")]; + tensor var_912 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_912")]; + tensor filled_window_21 = add(x = var_911, y = var_912)[name = tensor("filled_window_21")]; + tensor var_914 = greater_equal(x = clip_19, y = var_16)[name = tensor("op_914")]; + tensor var_915_dtype_0 = const()[name = tensor("op_915_dtype_0"), val = tensor("fp32")]; + tensor var_916 = const()[name = tensor("op_916"), val = tensor([1, 1, 1])]; + tensor var_915 = cast(dtype = var_915_dtype_0, x = var_914)[name = tensor("cast_154")]; + tensor use_roll_21 = reshape(shape = var_916, x = var_915)[name = tensor("use_roll_21")]; + tensor var_918 = sub(x = var_17, y = use_roll_21)[name = tensor("op_918")]; + tensor var_919 = mul(x = filled_window_21, y = var_918)[name = tensor("op_919")]; + tensor var_920 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_920")]; + tensor window_27 = add(x = var_919, y = var_920)[name = tensor("window_27")]; + tensor var_922 = const()[name = tensor("op_922"), val = tensor(0x1p+0)]; + tensor var_923 = add(x = clip_19, y = var_922)[name = tensor("op_923")]; + tensor clip_21 = clip(alpha = var_21, beta = var_16, x = var_923)[name = tensor("clip_21")]; + tensor var_926_begin_0 = const()[name = tensor("op_926_begin_0"), val = tensor([0, 3, 0])]; + tensor var_926_end_0 = const()[name = tensor("op_926_end_0"), val = tensor([1, 1, 256])]; + tensor var_926_end_mask_0 = const()[name = tensor("op_926_end_mask_0"), val = tensor([true, true, true])]; + tensor var_926 = slice_by_index(begin = var_926_begin_0, end = var_926_end_0, end_mask = var_926_end_mask_0, x = x_15)[name = tensor("op_926")]; + tensor var_929_begin_0 = const()[name = tensor("op_929_begin_0"), val = tensor([0, 1, 0])]; + tensor var_929_end_0 = const()[name = tensor("op_929_end_0"), val = tensor([1, 16, 256])]; + tensor var_929_end_mask_0 = const()[name = tensor("op_929_end_mask_0"), val = tensor([true, true, true])]; + tensor var_929 = slice_by_index(begin = var_929_begin_0, end = var_929_end_0, end_mask = var_929_end_mask_0, x = window_27)[name = tensor("op_929")]; tensor rolled_23_interleave_0 = const()[name = tensor("rolled_23_interleave_0"), val = tensor(false)]; - tensor rolled_23 = concat(axis = var_33, interleave = rolled_23_interleave_0, values = (var_1413, var_1410))[name = tensor("rolled_23")]; + tensor rolled_23 = concat(axis = var_33, interleave = rolled_23_interleave_0, values = (var_929, var_926))[name = tensor("rolled_23")]; tensor clip_22 = clip(alpha = var_21, beta = var_20, x = clip_21)[name = tensor("clip_22")]; - tensor var_1418 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_1418")]; - tensor var_1419_dtype_0 = const()[name = tensor("op_1419_dtype_0"), val = tensor("fp32")]; - tensor var_1420 = const()[name = tensor("op_1420"), val = tensor([1, 16, 1])]; - tensor var_1419 = cast(dtype = var_1419_dtype_0, x = var_1418)[name = tensor("cast_153")]; - tensor fill_mask_23 = reshape(shape = var_1420, x = var_1419)[name = tensor("fill_mask_23")]; + tensor var_934 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_934")]; + tensor var_935_dtype_0 = const()[name = tensor("op_935_dtype_0"), val = tensor("fp32")]; + tensor var_936 = const()[name = tensor("op_936"), val = tensor([1, 16, 1])]; + tensor var_935 = cast(dtype = var_935_dtype_0, x = var_934)[name = tensor("cast_153")]; + tensor fill_mask_23 = reshape(shape = var_936, x = var_935)[name = tensor("fill_mask_23")]; tensor x_expanded_23_reps_0 = const()[name = tensor("x_expanded_23_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_23 = tile(reps = x_expanded_23_reps_0, x = var_1410)[name = tensor("x_expanded_23")]; - tensor var_1424 = sub(x = var_17, y = fill_mask_23)[name = tensor("op_1424")]; - tensor var_1425 = mul(x = input_181, y = var_1424)[name = tensor("op_1425")]; - tensor var_1426 = mul(x = x_expanded_23, y = fill_mask_23)[name = tensor("op_1426")]; - tensor filled_window_23 = add(x = var_1425, y = var_1426)[name = tensor("filled_window_23")]; - tensor var_1428 = greater_equal(x = clip_21, y = var_16)[name = tensor("op_1428")]; - tensor var_1429_dtype_0 = const()[name = tensor("op_1429_dtype_0"), val = tensor("fp32")]; - tensor var_1430 = const()[name = tensor("op_1430"), val = tensor([1, 1, 1])]; - tensor var_1429 = cast(dtype = var_1429_dtype_0, x = var_1428)[name = tensor("cast_152")]; - tensor use_roll_23 = reshape(shape = var_1430, x = var_1429)[name = tensor("use_roll_23")]; - tensor var_1432 = sub(x = var_17, y = use_roll_23)[name = tensor("op_1432")]; - tensor var_1433 = mul(x = filled_window_23, y = var_1432)[name = tensor("op_1433")]; - tensor var_1434 = mul(x = rolled_23, y = use_roll_23)[name = tensor("op_1434")]; - tensor input_191 = add(x = var_1433, y = var_1434)[name = tensor("input_191")]; - tensor var_1436 = const()[name = tensor("op_1436"), val = tensor(0x1p+0)]; - tensor var_1437 = add(x = clip_21, y = var_1436)[name = tensor("op_1437")]; - tensor clip_23 = clip(alpha = var_21, beta = var_16, x = var_1437)[name = tensor("clip_23")]; - tensor x_35_axes_0 = const()[name = tensor("x_35_axes_0"), val = tensor([-1])]; - tensor x_35 = layer_norm(axes = x_35_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_191)[name = tensor("x_35")]; - tensor input_193_perm_0 = const()[name = tensor("input_193_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_77_pad_type_0 = const()[name = tensor("inputs_77_pad_type_0"), val = tensor("valid")]; - tensor inputs_77_strides_0 = const()[name = tensor("inputs_77_strides_0"), val = tensor([1])]; - tensor inputs_77_pad_0 = const()[name = tensor("inputs_77_pad_0"), val = tensor([0, 0])]; - tensor inputs_77_dilations_0 = const()[name = tensor("inputs_77_dilations_0"), val = tensor([1])]; - tensor inputs_77_groups_0 = const()[name = tensor("inputs_77_groups_0"), val = tensor(1)]; - tensor input_193 = transpose(perm = input_193_perm_0, x = x_35)[name = tensor("transpose_44")]; - tensor inputs_77 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_77_dilations_0, groups = inputs_77_groups_0, pad = inputs_77_pad_0, pad_type = inputs_77_pad_type_0, strides = inputs_77_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_193)[name = tensor("inputs_77")]; - tensor var_1458_split_sizes_0 = const()[name = tensor("op_1458_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1458_axis_0 = const()[name = tensor("op_1458_axis_0"), val = tensor(1)]; - tensor var_1458_0, tensor var_1458_1 = split(axis = var_1458_axis_0, split_sizes = var_1458_split_sizes_0, x = inputs_77)[name = tensor("op_1458")]; - tensor var_1460 = sigmoid(x = var_1458_1)[name = tensor("op_1460")]; - tensor inputs_79 = mul(x = var_1458_0, y = var_1460)[name = tensor("inputs_79")]; - tensor outputs_aug_23_pad_type_0 = const()[name = tensor("outputs_aug_23_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_23_pad_0 = const()[name = tensor("outputs_aug_23_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_23_groups_0 = const()[name = tensor("outputs_aug_23_groups_0"), val = tensor(256)]; - tensor outputs_aug_23_strides_0 = const()[name = tensor("outputs_aug_23_strides_0"), val = tensor([1])]; - tensor outputs_aug_23_dilations_0 = const()[name = tensor("outputs_aug_23_dilations_0"), val = tensor([1])]; - tensor outputs_aug_23 = conv(dilations = outputs_aug_23_dilations_0, groups = outputs_aug_23_groups_0, pad = outputs_aug_23_pad_0, pad_type = outputs_aug_23_pad_type_0, strides = outputs_aug_23_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_79)[name = tensor("outputs_aug_23")]; - tensor input_195_begin_0 = const()[name = tensor("input_195_begin_0"), val = tensor([0, 0, 0])]; - tensor input_195_end_0 = const()[name = tensor("input_195_end_0"), val = tensor([1, 256, 16])]; - tensor input_195_end_mask_0 = const()[name = tensor("input_195_end_mask_0"), val = tensor([true, true, false])]; - tensor input_195 = slice_by_index(begin = input_195_begin_0, end = input_195_end_0, end_mask = input_195_end_mask_0, x = outputs_aug_23)[name = tensor("input_195")]; - tensor inputs_81 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_195)[name = tensor("inputs_81")]; - tensor input_197 = silu(x = inputs_81)[name = tensor("input_197")]; - tensor input_199_pad_type_0 = const()[name = tensor("input_199_pad_type_0"), val = tensor("valid")]; - tensor input_199_strides_0 = const()[name = tensor("input_199_strides_0"), val = tensor([1])]; - tensor input_199_pad_0 = const()[name = tensor("input_199_pad_0"), val = tensor([0, 0])]; - tensor input_199_dilations_0 = const()[name = tensor("input_199_dilations_0"), val = tensor([1])]; - tensor input_199_groups_0 = const()[name = tensor("input_199_groups_0"), val = tensor(1)]; - tensor input_199 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_199_dilations_0, groups = input_199_groups_0, pad = input_199_pad_0, pad_type = input_199_pad_type_0, strides = input_199_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_197)[name = tensor("input_199")]; - tensor conv_out_23_perm_0 = const()[name = tensor("conv_out_23_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1491_begin_0 = const()[name = tensor("op_1491_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1491_end_0 = const()[name = tensor("op_1491_end_0"), val = tensor([1, 16, 256])]; - tensor var_1491_end_mask_0 = const()[name = tensor("op_1491_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_23 = transpose(perm = conv_out_23_perm_0, x = input_199)[name = tensor("transpose_43")]; - tensor var_1491 = slice_by_index(begin = var_1491_begin_0, end = var_1491_end_0, end_mask = var_1491_end_mask_0, x = conv_out_23)[name = tensor("op_1491")]; - tensor var_1494_interleave_0 = const()[name = tensor("op_1494_interleave_0"), val = tensor(false)]; - tensor var_1494 = concat(axis = var_33, interleave = var_1494_interleave_0, values = (var_1239, var_1323, var_1407, var_1491))[name = tensor("op_1494")]; - tensor input_201 = add(x = x_27, y = var_1494)[name = tensor("input_201")]; - tensor input_203_axes_0 = const()[name = tensor("input_203_axes_0"), val = tensor([-1])]; - tensor input_203 = layer_norm(axes = input_203_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_201)[name = tensor("input_203")]; - tensor inputs_83 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_203)[name = tensor("linear_26")]; - tensor input_205 = silu(x = inputs_83)[name = tensor("input_205")]; - tensor input_209 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_205)[name = tensor("linear_27")]; - tensor var_1517 = const()[name = tensor("op_1517"), val = tensor(0x1p-1)]; - tensor var_1518 = mul(x = input_209, y = var_1517)[name = tensor("op_1518")]; - tensor input_211 = add(x = var_1518, y = input_201)[name = tensor("input_211")]; - tensor input_213_axes_0 = const()[name = tensor("input_213_axes_0"), val = tensor([-1])]; - tensor input_213 = layer_norm(axes = input_213_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_211)[name = tensor("input_213")]; - tensor input_215_axes_0 = const()[name = tensor("input_215_axes_0"), val = tensor([-1])]; - tensor input_215 = layer_norm(axes = input_215_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_213)[name = tensor("input_215")]; - tensor inputs_85 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_215)[name = tensor("linear_28")]; - tensor input_217 = silu(x = inputs_85)[name = tensor("input_217")]; - tensor input_221 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_217)[name = tensor("linear_29")]; - tensor var_1547 = const()[name = tensor("op_1547"), val = tensor(0x1p-1)]; - tensor var_1548 = mul(x = input_221, y = var_1547)[name = tensor("op_1548")]; - tensor input_223 = add(x = var_1548, y = input_213)[name = tensor("input_223")]; - tensor x_37_axes_0 = const()[name = tensor("x_37_axes_0"), val = tensor([-1])]; - tensor x_37 = layer_norm(axes = x_37_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_223)[name = tensor("x_37")]; + tensor x_expanded_23 = tile(reps = x_expanded_23_reps_0, x = var_926)[name = tensor("x_expanded_23")]; + tensor var_940 = sub(x = var_17, y = fill_mask_23)[name = tensor("op_940")]; + tensor var_941 = mul(x = window_27, y = var_940)[name = tensor("op_941")]; + tensor var_942 = mul(x = x_expanded_23, y = fill_mask_23)[name = tensor("op_942")]; + tensor filled_window_23 = add(x = var_941, y = var_942)[name = tensor("filled_window_23")]; + tensor var_944 = greater_equal(x = clip_21, y = var_16)[name = tensor("op_944")]; + tensor var_945_dtype_0 = const()[name = tensor("op_945_dtype_0"), val = tensor("fp32")]; + tensor var_946 = const()[name = tensor("op_946"), val = tensor([1, 1, 1])]; + tensor var_945 = cast(dtype = var_945_dtype_0, x = var_944)[name = tensor("cast_152")]; + tensor use_roll_23 = reshape(shape = var_946, x = var_945)[name = tensor("use_roll_23")]; + tensor var_948 = sub(x = var_17, y = use_roll_23)[name = tensor("op_948")]; + tensor var_949 = mul(x = filled_window_23, y = var_948)[name = tensor("op_949")]; + tensor var_950 = mul(x = rolled_23, y = use_roll_23)[name = tensor("op_950")]; + tensor window_29 = add(x = var_949, y = var_950)[name = tensor("window_29")]; + tensor var_952 = const()[name = tensor("op_952"), val = tensor(0x1p+0)]; + tensor var_953 = add(x = clip_21, y = var_952)[name = tensor("op_953")]; + tensor clip_23 = clip(alpha = var_21, beta = var_16, x = var_953)[name = tensor("clip_23")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_30, interleave = input_101_interleave_0, values = (window_23, window_25, window_27, window_29))[name = tensor("input_101")]; + tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; + tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; + tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_23_pad_type_0 = const()[name = tensor("inputs_23_pad_type_0"), val = tensor("valid")]; + tensor inputs_23_strides_0 = const()[name = tensor("inputs_23_strides_0"), val = tensor([1])]; + tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; + tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; + tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; + tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; + tensor var_976_split_sizes_0 = const()[name = tensor("op_976_split_sizes_0"), val = tensor([256, 256])]; + tensor var_976_axis_0 = const()[name = tensor("op_976_axis_0"), val = tensor(1)]; + tensor var_976_0, tensor var_976_1 = split(axis = var_976_axis_0, split_sizes = var_976_split_sizes_0, x = inputs_23)[name = tensor("op_976")]; + tensor var_978 = sigmoid(x = var_976_1)[name = tensor("op_978")]; + tensor inputs_25 = mul(x = var_976_0, y = var_978)[name = tensor("inputs_25")]; + tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; + tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; + tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; + tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_25)[name = tensor("outputs_aug_5")]; + tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; + tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([4, 256, 16])]; + tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; + tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_5)[name = tensor("input_105")]; + tensor inputs_27 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_105)[name = tensor("inputs_27")]; + tensor input_107 = silu(x = inputs_27)[name = tensor("input_107")]; + tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; + tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; + tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; + tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; + tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; + tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; + tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1009_begin_0 = const()[name = tensor("op_1009_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1009_end_0 = const()[name = tensor("op_1009_end_0"), val = tensor([4, 16, 256])]; + tensor var_1009_end_mask_0 = const()[name = tensor("op_1009_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_1009 = slice_by_index(begin = var_1009_begin_0, end = var_1009_end_0, end_mask = var_1009_end_mask_0, x = conv_out_5)[name = tensor("op_1009")]; + tensor var_1011_perm_0 = const()[name = tensor("op_1011_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1011 = transpose(perm = var_1011_perm_0, x = var_1009)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_1011)[name = tensor("input_111")]; + tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; + tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; + tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; + tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; + tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; + tensor var_1034 = const()[name = tensor("op_1034"), val = tensor(0x1p-1)]; + tensor var_1035 = mul(x = input_119, y = var_1034)[name = tensor("op_1035")]; + tensor input_121 = add(x = var_1035, y = input_111)[name = tensor("input_121")]; + tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; + tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; + tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; + tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; + tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; + tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; + tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; + tensor var_1064 = const()[name = tensor("op_1064"), val = tensor(0x1p-1)]; + tensor var_1065 = mul(x = input_131, y = var_1064)[name = tensor("op_1065")]; + tensor input_133 = add(x = var_1065, y = input_123)[name = tensor("input_133")]; + tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; + tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; tensor prev_kv_7_end_0 = const()[name = tensor("prev_kv_7_end_0"), val = tensor([4, 1, 4, 64, 64])]; tensor prev_kv_7_end_mask_0 = const()[name = tensor("prev_kv_7_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1422,382 +1078,267 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_1562 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_37)[name = tensor("linear_30")]; - tensor var_1563 = const()[name = tensor("op_1563"), val = tensor([1, 4, 4, 64])]; - tensor var_1564 = reshape(shape = var_1563, x = var_1562)[name = tensor("op_1564")]; + tensor var_1079 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_1080 = const()[name = tensor("op_1080"), val = tensor([1, 4, 4, 64])]; + tensor var_1081 = reshape(shape = var_1080, x = var_1079)[name = tensor("op_1081")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1568 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_37)[name = tensor("linear_31")]; - tensor var_1569 = const()[name = tensor("op_1569"), val = tensor(0x1p-3)]; - tensor var_1570 = mul(x = var_1568, y = var_1569)[name = tensor("op_1570")]; - tensor var_1571 = const()[name = tensor("op_1571"), val = tensor([1, 4, 4, 64])]; - tensor var_1572 = reshape(shape = var_1571, x = var_1570)[name = tensor("op_1572")]; + tensor var_1085 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_1086 = const()[name = tensor("op_1086"), val = tensor(0x1p-3)]; + tensor var_1087 = mul(x = var_1085, y = var_1086)[name = tensor("op_1087")]; + tensor var_1088 = const()[name = tensor("op_1088"), val = tensor([1, 4, 4, 64])]; + tensor var_1089 = reshape(shape = var_1088, x = var_1087)[name = tensor("op_1089")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1576 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_37)[name = tensor("linear_32")]; - tensor var_1577 = const()[name = tensor("op_1577"), val = tensor([1, 4, 4, 64])]; - tensor var_1578 = reshape(shape = var_1577, x = var_1576)[name = tensor("op_1578")]; + tensor var_1093 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_1094 = const()[name = tensor("op_1094"), val = tensor([1, 4, 4, 64])]; + tensor var_1095 = reshape(shape = var_1094, x = var_1093)[name = tensor("op_1095")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_227 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_37)[name = tensor("linear_33")]; + tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; tensor s_t_7 = add(x = prev_scale_7, y = encoder__t_index)[name = tensor("s_t_7")]; tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_1572)[name = tensor("transpose_41")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_1564)[name = tensor("transpose_42")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_1089)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_1081)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_1588 = const()[name = tensor("op_1588"), val = tensor([4, 1])]; - tensor var_1589 = reshape(shape = var_1588, x = sqrt_s_t_7)[name = tensor("op_1589")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_1589)[name = tensor("M_7")]; - tensor var_1591 = mul(x = qk_7, y = M_7)[name = tensor("op_1591")]; + tensor var_1105 = const()[name = tensor("op_1105"), val = tensor([4, 1])]; + tensor var_1106 = reshape(shape = var_1105, x = sqrt_s_t_7)[name = tensor("op_1106")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_1106)[name = tensor("M_7")]; + tensor var_1108 = mul(x = qk_7, y = M_7)[name = tensor("op_1108")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_1578)[name = tensor("transpose_40")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1591, y = v_7)[name = tensor("inner_7")]; - tensor var_1593_transpose_x_0 = const()[name = tensor("op_1593_transpose_x_0"), val = tensor(false)]; - tensor var_1593_transpose_y_0 = const()[name = tensor("op_1593_transpose_y_0"), val = tensor(false)]; - tensor var_1593 = matmul(transpose_x = var_1593_transpose_x_0, transpose_y = var_1593_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1593")]; - tensor var_1594 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1594")]; - tensor var_1595 = const()[name = tensor("op_1595"), val = tensor([1, 1, 4, 1])]; - tensor var_1596 = reshape(shape = var_1595, x = var_1594)[name = tensor("op_1596")]; - tensor cross_7 = mul(x = var_1593, y = var_1596)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_1095)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1108, y = v_7)[name = tensor("inner_7")]; + tensor var_1110_transpose_x_0 = const()[name = tensor("op_1110_transpose_x_0"), val = tensor(false)]; + tensor var_1110_transpose_y_0 = const()[name = tensor("op_1110_transpose_y_0"), val = tensor(false)]; + tensor var_1110 = matmul(transpose_x = var_1110_transpose_x_0, transpose_y = var_1110_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1110")]; + tensor var_1111 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1111")]; + tensor var_1112 = const()[name = tensor("op_1112"), val = tensor([1, 1, 4, 1])]; + tensor var_1113 = reshape(shape = var_1112, x = var_1111)[name = tensor("op_1113")]; + tensor cross_7 = mul(x = var_1110, y = var_1113)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_1599 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1599")]; - tensor var_1601_transpose_x_1 = const()[name = tensor("op_1601_transpose_x_1"), val = tensor(true)]; - tensor var_1601_transpose_y_1 = const()[name = tensor("op_1601_transpose_y_1"), val = tensor(false)]; - tensor var_1601 = matmul(transpose_x = var_1601_transpose_x_1, transpose_y = var_1601_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1601")]; - tensor new_kv_unnorm_7 = add(x = var_1599, y = var_1601)[name = tensor("new_kv_unnorm_7")]; - tensor var_1603 = const()[name = tensor("op_1603"), val = tensor(0x1p+2)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_1603)[name = tensor("new_scale_7")]; - tensor var_1605 = sqrt(x = new_scale_7)[name = tensor("op_1605")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1605)[name = tensor("nkv_1")]; - tensor var_1607_perm_0 = const()[name = tensor("op_1607_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1116 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1116")]; + tensor var_1118_transpose_x_1 = const()[name = tensor("op_1118_transpose_x_1"), val = tensor(true)]; + tensor var_1118_transpose_y_1 = const()[name = tensor("op_1118_transpose_y_1"), val = tensor(false)]; + tensor var_1118 = matmul(transpose_x = var_1118_transpose_x_1, transpose_y = var_1118_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1118")]; + tensor new_kv_unnorm_7 = add(x = var_1116, y = var_1118)[name = tensor("new_kv_unnorm_7")]; + tensor var_1120 = const()[name = tensor("op_1120"), val = tensor(0x1p+2)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_1120)[name = tensor("new_scale_7")]; + tensor var_1122 = sqrt(x = new_scale_7)[name = tensor("op_1122")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1122)[name = tensor("nkv_1")]; + tensor var_1124_perm_0 = const()[name = tensor("op_1124_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_1607 = transpose(perm = var_1607_perm_0, x = out_19)[name = tensor("transpose_39")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1607)[name = tensor("out_21")]; - tensor var_1611 = const()[name = tensor("op_1611"), val = tensor([1, 4, 256])]; - tensor out_23 = reshape(shape = var_1611, x = out_21)[name = tensor("out_23")]; - tensor var_1613 = silu(x = input_227)[name = tensor("op_1613")]; - tensor input_229 = mul(x = var_1613, y = out_23)[name = tensor("input_229")]; - tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_229)[name = tensor("linear_34")]; - tensor x_39 = add(x = input_223, y = ret_out_7)[name = tensor("x_39")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor var_1124 = transpose(perm = var_1124_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1124)[name = tensor("out_21")]; + tensor var_1128 = const()[name = tensor("op_1128"), val = tensor([1, 4, 256])]; + tensor out_23 = reshape(shape = var_1128, x = out_21)[name = tensor("out_23")]; + tensor var_1130 = silu(x = input_137)[name = tensor("op_1130")]; + tensor input_139 = mul(x = var_1130, y = out_23)[name = tensor("input_139")]; + tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; + tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; + tensor window_31_begin_0 = const()[name = tensor("window_31_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_31_end_0 = const()[name = tensor("window_31_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_31_end_mask_0 = const()[name = tensor("window_31_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_31_squeeze_mask_0 = const()[name = tensor("window_31_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_31 = slice_by_index(begin = window_31_begin_0, end = window_31_end_0, end_mask = window_31_end_mask_0, squeeze_mask = window_31_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_31")]; tensor filled_25_begin_0 = const()[name = tensor("filled_25_begin_0"), val = tensor([3, 0])]; tensor filled_25_end_0 = const()[name = tensor("filled_25_end_0"), val = tensor([4, 1])]; tensor filled_25_end_mask_0 = const()[name = tensor("filled_25_end_mask_0"), val = tensor([false, true])]; tensor filled_25_squeeze_mask_0 = const()[name = tensor("filled_25_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_25 = slice_by_index(begin = filled_25_begin_0, end = filled_25_end_0, end_mask = filled_25_end_mask_0, squeeze_mask = filled_25_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_25")]; - tensor var_1622_begin_0 = const()[name = tensor("op_1622_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1622_end_0 = const()[name = tensor("op_1622_end_0"), val = tensor([1, 1, 256])]; - tensor var_1622_end_mask_0 = const()[name = tensor("op_1622_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1622 = slice_by_index(begin = var_1622_begin_0, end = var_1622_end_0, end_mask = var_1622_end_mask_0, x = x_39)[name = tensor("op_1622")]; - tensor var_1625_begin_0 = const()[name = tensor("op_1625_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1625_end_0 = const()[name = tensor("op_1625_end_0"), val = tensor([1, 16, 256])]; - tensor var_1625_end_mask_0 = const()[name = tensor("op_1625_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1625 = slice_by_index(begin = var_1625_begin_0, end = var_1625_end_0, end_mask = var_1625_end_mask_0, x = window)[name = tensor("op_1625")]; + tensor var_1139_begin_0 = const()[name = tensor("op_1139_begin_0"), val = tensor([0, 0, 0])]; + tensor var_1139_end_0 = const()[name = tensor("op_1139_end_0"), val = tensor([1, 1, 256])]; + tensor var_1139_end_mask_0 = const()[name = tensor("op_1139_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1139 = slice_by_index(begin = var_1139_begin_0, end = var_1139_end_0, end_mask = var_1139_end_mask_0, x = x_21)[name = tensor("op_1139")]; + tensor var_1142_begin_0 = const()[name = tensor("op_1142_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1142_end_0 = const()[name = tensor("op_1142_end_0"), val = tensor([1, 16, 256])]; + tensor var_1142_end_mask_0 = const()[name = tensor("op_1142_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1142 = slice_by_index(begin = var_1142_begin_0, end = var_1142_end_0, end_mask = var_1142_end_mask_0, x = window_31)[name = tensor("op_1142")]; tensor rolled_25_interleave_0 = const()[name = tensor("rolled_25_interleave_0"), val = tensor(false)]; - tensor rolled_25 = concat(axis = var_33, interleave = rolled_25_interleave_0, values = (var_1625, var_1622))[name = tensor("rolled_25")]; + tensor rolled_25 = concat(axis = var_33, interleave = rolled_25_interleave_0, values = (var_1142, var_1139))[name = tensor("rolled_25")]; tensor clip_24 = clip(alpha = var_21, beta = var_20, x = filled_25)[name = tensor("clip_24")]; - tensor var_1630 = equal(x = encoder__conv_positions, y = clip_24)[name = tensor("op_1630")]; - tensor var_1631_dtype_0 = const()[name = tensor("op_1631_dtype_0"), val = tensor("fp32")]; - tensor var_1632 = const()[name = tensor("op_1632"), val = tensor([1, 16, 1])]; - tensor var_1631 = cast(dtype = var_1631_dtype_0, x = var_1630)[name = tensor("cast_151")]; - tensor fill_mask_25 = reshape(shape = var_1632, x = var_1631)[name = tensor("fill_mask_25")]; + tensor var_1147 = equal(x = encoder__conv_positions, y = clip_24)[name = tensor("op_1147")]; + tensor var_1148_dtype_0 = const()[name = tensor("op_1148_dtype_0"), val = tensor("fp32")]; + tensor var_1149 = const()[name = tensor("op_1149"), val = tensor([1, 16, 1])]; + tensor var_1148 = cast(dtype = var_1148_dtype_0, x = var_1147)[name = tensor("cast_151")]; + tensor fill_mask_25 = reshape(shape = var_1149, x = var_1148)[name = tensor("fill_mask_25")]; tensor x_expanded_25_reps_0 = const()[name = tensor("x_expanded_25_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_25 = tile(reps = x_expanded_25_reps_0, x = var_1622)[name = tensor("x_expanded_25")]; - tensor var_1636 = sub(x = var_17, y = fill_mask_25)[name = tensor("op_1636")]; - tensor var_1637 = mul(x = window, y = var_1636)[name = tensor("op_1637")]; - tensor var_1638 = mul(x = x_expanded_25, y = fill_mask_25)[name = tensor("op_1638")]; - tensor filled_window_25 = add(x = var_1637, y = var_1638)[name = tensor("filled_window_25")]; - tensor var_1640 = greater_equal(x = filled_25, y = var_16)[name = tensor("op_1640")]; - tensor var_1641_dtype_0 = const()[name = tensor("op_1641_dtype_0"), val = tensor("fp32")]; - tensor var_1642 = const()[name = tensor("op_1642"), val = tensor([1, 1, 1])]; - tensor var_1641 = cast(dtype = var_1641_dtype_0, x = var_1640)[name = tensor("cast_150")]; - tensor use_roll_25 = reshape(shape = var_1642, x = var_1641)[name = tensor("use_roll_25")]; - tensor var_1644 = sub(x = var_17, y = use_roll_25)[name = tensor("op_1644")]; - tensor var_1645 = mul(x = filled_window_25, y = var_1644)[name = tensor("op_1645")]; - tensor var_1646 = mul(x = rolled_25, y = use_roll_25)[name = tensor("op_1646")]; - tensor input_231 = add(x = var_1645, y = var_1646)[name = tensor("input_231")]; - tensor var_1648 = const()[name = tensor("op_1648"), val = tensor(0x1p+0)]; - tensor var_1649 = add(x = filled_25, y = var_1648)[name = tensor("op_1649")]; - tensor clip_25 = clip(alpha = var_21, beta = var_16, x = var_1649)[name = tensor("clip_25")]; - tensor x_41_axes_0 = const()[name = tensor("x_41_axes_0"), val = tensor([-1])]; - tensor x_41 = layer_norm(axes = x_41_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_231)[name = tensor("x_41")]; - tensor input_233_perm_0 = const()[name = tensor("input_233_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_87_pad_type_0 = const()[name = tensor("inputs_87_pad_type_0"), val = tensor("valid")]; - tensor inputs_87_strides_0 = const()[name = tensor("inputs_87_strides_0"), val = tensor([1])]; - tensor inputs_87_pad_0 = const()[name = tensor("inputs_87_pad_0"), val = tensor([0, 0])]; - tensor inputs_87_dilations_0 = const()[name = tensor("inputs_87_dilations_0"), val = tensor([1])]; - tensor inputs_87_groups_0 = const()[name = tensor("inputs_87_groups_0"), val = tensor(1)]; - tensor input_233 = transpose(perm = input_233_perm_0, x = x_41)[name = tensor("transpose_38")]; - tensor inputs_87 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_87_dilations_0, groups = inputs_87_groups_0, pad = inputs_87_pad_0, pad_type = inputs_87_pad_type_0, strides = inputs_87_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_233)[name = tensor("inputs_87")]; - tensor var_1670_split_sizes_0 = const()[name = tensor("op_1670_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1670_axis_0 = const()[name = tensor("op_1670_axis_0"), val = tensor(1)]; - tensor var_1670_0, tensor var_1670_1 = split(axis = var_1670_axis_0, split_sizes = var_1670_split_sizes_0, x = inputs_87)[name = tensor("op_1670")]; - tensor var_1672 = sigmoid(x = var_1670_1)[name = tensor("op_1672")]; - tensor inputs_89 = mul(x = var_1670_0, y = var_1672)[name = tensor("inputs_89")]; - tensor outputs_aug_25_pad_type_0 = const()[name = tensor("outputs_aug_25_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_25_pad_0 = const()[name = tensor("outputs_aug_25_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_25_groups_0 = const()[name = tensor("outputs_aug_25_groups_0"), val = tensor(256)]; - tensor outputs_aug_25_strides_0 = const()[name = tensor("outputs_aug_25_strides_0"), val = tensor([1])]; - tensor outputs_aug_25_dilations_0 = const()[name = tensor("outputs_aug_25_dilations_0"), val = tensor([1])]; - tensor outputs_aug_25 = conv(dilations = outputs_aug_25_dilations_0, groups = outputs_aug_25_groups_0, pad = outputs_aug_25_pad_0, pad_type = outputs_aug_25_pad_type_0, strides = outputs_aug_25_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_89)[name = tensor("outputs_aug_25")]; - tensor input_235_begin_0 = const()[name = tensor("input_235_begin_0"), val = tensor([0, 0, 0])]; - tensor input_235_end_0 = const()[name = tensor("input_235_end_0"), val = tensor([1, 256, 16])]; - tensor input_235_end_mask_0 = const()[name = tensor("input_235_end_mask_0"), val = tensor([true, true, false])]; - tensor input_235 = slice_by_index(begin = input_235_begin_0, end = input_235_end_0, end_mask = input_235_end_mask_0, x = outputs_aug_25)[name = tensor("input_235")]; - tensor inputs_91 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_235)[name = tensor("inputs_91")]; - tensor input_237 = silu(x = inputs_91)[name = tensor("input_237")]; - tensor input_239_pad_type_0 = const()[name = tensor("input_239_pad_type_0"), val = tensor("valid")]; - tensor input_239_strides_0 = const()[name = tensor("input_239_strides_0"), val = tensor([1])]; - tensor input_239_pad_0 = const()[name = tensor("input_239_pad_0"), val = tensor([0, 0])]; - tensor input_239_dilations_0 = const()[name = tensor("input_239_dilations_0"), val = tensor([1])]; - tensor input_239_groups_0 = const()[name = tensor("input_239_groups_0"), val = tensor(1)]; - tensor input_239 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_239_dilations_0, groups = input_239_groups_0, pad = input_239_pad_0, pad_type = input_239_pad_type_0, strides = input_239_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_237)[name = tensor("input_239")]; - tensor conv_out_25_perm_0 = const()[name = tensor("conv_out_25_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1703_begin_0 = const()[name = tensor("op_1703_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1703_end_0 = const()[name = tensor("op_1703_end_0"), val = tensor([1, 16, 256])]; - tensor var_1703_end_mask_0 = const()[name = tensor("op_1703_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_25 = transpose(perm = conv_out_25_perm_0, x = input_239)[name = tensor("transpose_37")]; - tensor var_1703 = slice_by_index(begin = var_1703_begin_0, end = var_1703_end_0, end_mask = var_1703_end_mask_0, x = conv_out_25)[name = tensor("op_1703")]; - tensor var_1706_begin_0 = const()[name = tensor("op_1706_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1706_end_0 = const()[name = tensor("op_1706_end_0"), val = tensor([1, 2, 256])]; - tensor var_1706_end_mask_0 = const()[name = tensor("op_1706_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1706 = slice_by_index(begin = var_1706_begin_0, end = var_1706_end_0, end_mask = var_1706_end_mask_0, x = x_39)[name = tensor("op_1706")]; - tensor var_1709_begin_0 = const()[name = tensor("op_1709_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1709_end_0 = const()[name = tensor("op_1709_end_0"), val = tensor([1, 16, 256])]; - tensor var_1709_end_mask_0 = const()[name = tensor("op_1709_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1709 = slice_by_index(begin = var_1709_begin_0, end = var_1709_end_0, end_mask = var_1709_end_mask_0, x = input_231)[name = tensor("op_1709")]; + tensor x_expanded_25 = tile(reps = x_expanded_25_reps_0, x = var_1139)[name = tensor("x_expanded_25")]; + tensor var_1153 = sub(x = var_17, y = fill_mask_25)[name = tensor("op_1153")]; + tensor var_1154 = mul(x = window_31, y = var_1153)[name = tensor("op_1154")]; + tensor var_1155 = mul(x = x_expanded_25, y = fill_mask_25)[name = tensor("op_1155")]; + tensor filled_window_25 = add(x = var_1154, y = var_1155)[name = tensor("filled_window_25")]; + tensor var_1157 = greater_equal(x = filled_25, y = var_16)[name = tensor("op_1157")]; + tensor var_1158_dtype_0 = const()[name = tensor("op_1158_dtype_0"), val = tensor("fp32")]; + tensor var_1159 = const()[name = tensor("op_1159"), val = tensor([1, 1, 1])]; + tensor var_1158 = cast(dtype = var_1158_dtype_0, x = var_1157)[name = tensor("cast_150")]; + tensor use_roll_25 = reshape(shape = var_1159, x = var_1158)[name = tensor("use_roll_25")]; + tensor var_1161 = sub(x = var_17, y = use_roll_25)[name = tensor("op_1161")]; + tensor var_1162 = mul(x = filled_window_25, y = var_1161)[name = tensor("op_1162")]; + tensor var_1163 = mul(x = rolled_25, y = use_roll_25)[name = tensor("op_1163")]; + tensor window_33 = add(x = var_1162, y = var_1163)[name = tensor("window_33")]; + tensor var_1165 = const()[name = tensor("op_1165"), val = tensor(0x1p+0)]; + tensor var_1166 = add(x = filled_25, y = var_1165)[name = tensor("op_1166")]; + tensor clip_25 = clip(alpha = var_21, beta = var_16, x = var_1166)[name = tensor("clip_25")]; + tensor var_1169_begin_0 = const()[name = tensor("op_1169_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1169_end_0 = const()[name = tensor("op_1169_end_0"), val = tensor([1, 2, 256])]; + tensor var_1169_end_mask_0 = const()[name = tensor("op_1169_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1169 = slice_by_index(begin = var_1169_begin_0, end = var_1169_end_0, end_mask = var_1169_end_mask_0, x = x_21)[name = tensor("op_1169")]; + tensor var_1172_begin_0 = const()[name = tensor("op_1172_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1172_end_0 = const()[name = tensor("op_1172_end_0"), val = tensor([1, 16, 256])]; + tensor var_1172_end_mask_0 = const()[name = tensor("op_1172_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1172 = slice_by_index(begin = var_1172_begin_0, end = var_1172_end_0, end_mask = var_1172_end_mask_0, x = window_33)[name = tensor("op_1172")]; tensor rolled_27_interleave_0 = const()[name = tensor("rolled_27_interleave_0"), val = tensor(false)]; - tensor rolled_27 = concat(axis = var_33, interleave = rolled_27_interleave_0, values = (var_1709, var_1706))[name = tensor("rolled_27")]; + tensor rolled_27 = concat(axis = var_33, interleave = rolled_27_interleave_0, values = (var_1172, var_1169))[name = tensor("rolled_27")]; tensor clip_26 = clip(alpha = var_21, beta = var_20, x = clip_25)[name = tensor("clip_26")]; - tensor var_1714 = equal(x = encoder__conv_positions, y = clip_26)[name = tensor("op_1714")]; - tensor var_1715_dtype_0 = const()[name = tensor("op_1715_dtype_0"), val = tensor("fp32")]; - tensor var_1716 = const()[name = tensor("op_1716"), val = tensor([1, 16, 1])]; - tensor var_1715 = cast(dtype = var_1715_dtype_0, x = var_1714)[name = tensor("cast_149")]; - tensor fill_mask_27 = reshape(shape = var_1716, x = var_1715)[name = tensor("fill_mask_27")]; + tensor var_1177 = equal(x = encoder__conv_positions, y = clip_26)[name = tensor("op_1177")]; + tensor var_1178_dtype_0 = const()[name = tensor("op_1178_dtype_0"), val = tensor("fp32")]; + tensor var_1179 = const()[name = tensor("op_1179"), val = tensor([1, 16, 1])]; + tensor var_1178 = cast(dtype = var_1178_dtype_0, x = var_1177)[name = tensor("cast_149")]; + tensor fill_mask_27 = reshape(shape = var_1179, x = var_1178)[name = tensor("fill_mask_27")]; tensor x_expanded_27_reps_0 = const()[name = tensor("x_expanded_27_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_27 = tile(reps = x_expanded_27_reps_0, x = var_1706)[name = tensor("x_expanded_27")]; - tensor var_1720 = sub(x = var_17, y = fill_mask_27)[name = tensor("op_1720")]; - tensor var_1721 = mul(x = input_231, y = var_1720)[name = tensor("op_1721")]; - tensor var_1722 = mul(x = x_expanded_27, y = fill_mask_27)[name = tensor("op_1722")]; - tensor filled_window_27 = add(x = var_1721, y = var_1722)[name = tensor("filled_window_27")]; - tensor var_1724 = greater_equal(x = clip_25, y = var_16)[name = tensor("op_1724")]; - tensor var_1725_dtype_0 = const()[name = tensor("op_1725_dtype_0"), val = tensor("fp32")]; - tensor var_1726 = const()[name = tensor("op_1726"), val = tensor([1, 1, 1])]; - tensor var_1725 = cast(dtype = var_1725_dtype_0, x = var_1724)[name = tensor("cast_148")]; - tensor use_roll_27 = reshape(shape = var_1726, x = var_1725)[name = tensor("use_roll_27")]; - tensor var_1728 = sub(x = var_17, y = use_roll_27)[name = tensor("op_1728")]; - tensor var_1729 = mul(x = filled_window_27, y = var_1728)[name = tensor("op_1729")]; - tensor var_1730 = mul(x = rolled_27, y = use_roll_27)[name = tensor("op_1730")]; - tensor input_241 = add(x = var_1729, y = var_1730)[name = tensor("input_241")]; - tensor var_1732 = const()[name = tensor("op_1732"), val = tensor(0x1p+0)]; - tensor var_1733 = add(x = clip_25, y = var_1732)[name = tensor("op_1733")]; - tensor clip_27 = clip(alpha = var_21, beta = var_16, x = var_1733)[name = tensor("clip_27")]; - tensor x_43_axes_0 = const()[name = tensor("x_43_axes_0"), val = tensor([-1])]; - tensor x_43 = layer_norm(axes = x_43_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_241)[name = tensor("x_43")]; - tensor input_243_perm_0 = const()[name = tensor("input_243_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_93_pad_type_0 = const()[name = tensor("inputs_93_pad_type_0"), val = tensor("valid")]; - tensor inputs_93_strides_0 = const()[name = tensor("inputs_93_strides_0"), val = tensor([1])]; - tensor inputs_93_pad_0 = const()[name = tensor("inputs_93_pad_0"), val = tensor([0, 0])]; - tensor inputs_93_dilations_0 = const()[name = tensor("inputs_93_dilations_0"), val = tensor([1])]; - tensor inputs_93_groups_0 = const()[name = tensor("inputs_93_groups_0"), val = tensor(1)]; - tensor input_243 = transpose(perm = input_243_perm_0, x = x_43)[name = tensor("transpose_36")]; - tensor inputs_93 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_93_dilations_0, groups = inputs_93_groups_0, pad = inputs_93_pad_0, pad_type = inputs_93_pad_type_0, strides = inputs_93_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_243)[name = tensor("inputs_93")]; - tensor var_1754_split_sizes_0 = const()[name = tensor("op_1754_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1754_axis_0 = const()[name = tensor("op_1754_axis_0"), val = tensor(1)]; - tensor var_1754_0, tensor var_1754_1 = split(axis = var_1754_axis_0, split_sizes = var_1754_split_sizes_0, x = inputs_93)[name = tensor("op_1754")]; - tensor var_1756 = sigmoid(x = var_1754_1)[name = tensor("op_1756")]; - tensor inputs_95 = mul(x = var_1754_0, y = var_1756)[name = tensor("inputs_95")]; - tensor outputs_aug_27_pad_type_0 = const()[name = tensor("outputs_aug_27_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_27_pad_0 = const()[name = tensor("outputs_aug_27_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_27_groups_0 = const()[name = tensor("outputs_aug_27_groups_0"), val = tensor(256)]; - tensor outputs_aug_27_strides_0 = const()[name = tensor("outputs_aug_27_strides_0"), val = tensor([1])]; - tensor outputs_aug_27_dilations_0 = const()[name = tensor("outputs_aug_27_dilations_0"), val = tensor([1])]; - tensor outputs_aug_27 = conv(dilations = outputs_aug_27_dilations_0, groups = outputs_aug_27_groups_0, pad = outputs_aug_27_pad_0, pad_type = outputs_aug_27_pad_type_0, strides = outputs_aug_27_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_95)[name = tensor("outputs_aug_27")]; - tensor input_245_begin_0 = const()[name = tensor("input_245_begin_0"), val = tensor([0, 0, 0])]; - tensor input_245_end_0 = const()[name = tensor("input_245_end_0"), val = tensor([1, 256, 16])]; - tensor input_245_end_mask_0 = const()[name = tensor("input_245_end_mask_0"), val = tensor([true, true, false])]; - tensor input_245 = slice_by_index(begin = input_245_begin_0, end = input_245_end_0, end_mask = input_245_end_mask_0, x = outputs_aug_27)[name = tensor("input_245")]; - tensor inputs_97 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_245)[name = tensor("inputs_97")]; - tensor input_247 = silu(x = inputs_97)[name = tensor("input_247")]; - tensor input_249_pad_type_0 = const()[name = tensor("input_249_pad_type_0"), val = tensor("valid")]; - tensor input_249_strides_0 = const()[name = tensor("input_249_strides_0"), val = tensor([1])]; - tensor input_249_pad_0 = const()[name = tensor("input_249_pad_0"), val = tensor([0, 0])]; - tensor input_249_dilations_0 = const()[name = tensor("input_249_dilations_0"), val = tensor([1])]; - tensor input_249_groups_0 = const()[name = tensor("input_249_groups_0"), val = tensor(1)]; - tensor input_249 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_249_dilations_0, groups = input_249_groups_0, pad = input_249_pad_0, pad_type = input_249_pad_type_0, strides = input_249_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_247)[name = tensor("input_249")]; - tensor conv_out_27_perm_0 = const()[name = tensor("conv_out_27_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1787_begin_0 = const()[name = tensor("op_1787_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1787_end_0 = const()[name = tensor("op_1787_end_0"), val = tensor([1, 16, 256])]; - tensor var_1787_end_mask_0 = const()[name = tensor("op_1787_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_27 = transpose(perm = conv_out_27_perm_0, x = input_249)[name = tensor("transpose_35")]; - tensor var_1787 = slice_by_index(begin = var_1787_begin_0, end = var_1787_end_0, end_mask = var_1787_end_mask_0, x = conv_out_27)[name = tensor("op_1787")]; - tensor var_1790_begin_0 = const()[name = tensor("op_1790_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1790_end_0 = const()[name = tensor("op_1790_end_0"), val = tensor([1, 3, 256])]; - tensor var_1790_end_mask_0 = const()[name = tensor("op_1790_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1790 = slice_by_index(begin = var_1790_begin_0, end = var_1790_end_0, end_mask = var_1790_end_mask_0, x = x_39)[name = tensor("op_1790")]; - tensor var_1793_begin_0 = const()[name = tensor("op_1793_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1793_end_0 = const()[name = tensor("op_1793_end_0"), val = tensor([1, 16, 256])]; - tensor var_1793_end_mask_0 = const()[name = tensor("op_1793_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1793 = slice_by_index(begin = var_1793_begin_0, end = var_1793_end_0, end_mask = var_1793_end_mask_0, x = input_241)[name = tensor("op_1793")]; + tensor x_expanded_27 = tile(reps = x_expanded_27_reps_0, x = var_1169)[name = tensor("x_expanded_27")]; + tensor var_1183 = sub(x = var_17, y = fill_mask_27)[name = tensor("op_1183")]; + tensor var_1184 = mul(x = window_33, y = var_1183)[name = tensor("op_1184")]; + tensor var_1185 = mul(x = x_expanded_27, y = fill_mask_27)[name = tensor("op_1185")]; + tensor filled_window_27 = add(x = var_1184, y = var_1185)[name = tensor("filled_window_27")]; + tensor var_1187 = greater_equal(x = clip_25, y = var_16)[name = tensor("op_1187")]; + tensor var_1188_dtype_0 = const()[name = tensor("op_1188_dtype_0"), val = tensor("fp32")]; + tensor var_1189 = const()[name = tensor("op_1189"), val = tensor([1, 1, 1])]; + tensor var_1188 = cast(dtype = var_1188_dtype_0, x = var_1187)[name = tensor("cast_148")]; + tensor use_roll_27 = reshape(shape = var_1189, x = var_1188)[name = tensor("use_roll_27")]; + tensor var_1191 = sub(x = var_17, y = use_roll_27)[name = tensor("op_1191")]; + tensor var_1192 = mul(x = filled_window_27, y = var_1191)[name = tensor("op_1192")]; + tensor var_1193 = mul(x = rolled_27, y = use_roll_27)[name = tensor("op_1193")]; + tensor window_35 = add(x = var_1192, y = var_1193)[name = tensor("window_35")]; + tensor var_1195 = const()[name = tensor("op_1195"), val = tensor(0x1p+0)]; + tensor var_1196 = add(x = clip_25, y = var_1195)[name = tensor("op_1196")]; + tensor clip_27 = clip(alpha = var_21, beta = var_16, x = var_1196)[name = tensor("clip_27")]; + tensor var_1199_begin_0 = const()[name = tensor("op_1199_begin_0"), val = tensor([0, 2, 0])]; + tensor var_1199_end_0 = const()[name = tensor("op_1199_end_0"), val = tensor([1, 3, 256])]; + tensor var_1199_end_mask_0 = const()[name = tensor("op_1199_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1199 = slice_by_index(begin = var_1199_begin_0, end = var_1199_end_0, end_mask = var_1199_end_mask_0, x = x_21)[name = tensor("op_1199")]; + tensor var_1202_begin_0 = const()[name = tensor("op_1202_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1202_end_0 = const()[name = tensor("op_1202_end_0"), val = tensor([1, 16, 256])]; + tensor var_1202_end_mask_0 = const()[name = tensor("op_1202_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1202 = slice_by_index(begin = var_1202_begin_0, end = var_1202_end_0, end_mask = var_1202_end_mask_0, x = window_35)[name = tensor("op_1202")]; tensor rolled_29_interleave_0 = const()[name = tensor("rolled_29_interleave_0"), val = tensor(false)]; - tensor rolled_29 = concat(axis = var_33, interleave = rolled_29_interleave_0, values = (var_1793, var_1790))[name = tensor("rolled_29")]; + tensor rolled_29 = concat(axis = var_33, interleave = rolled_29_interleave_0, values = (var_1202, var_1199))[name = tensor("rolled_29")]; tensor clip_28 = clip(alpha = var_21, beta = var_20, x = clip_27)[name = tensor("clip_28")]; - tensor var_1798 = equal(x = encoder__conv_positions, y = clip_28)[name = tensor("op_1798")]; - tensor var_1799_dtype_0 = const()[name = tensor("op_1799_dtype_0"), val = tensor("fp32")]; - tensor var_1800 = const()[name = tensor("op_1800"), val = tensor([1, 16, 1])]; - tensor var_1799 = cast(dtype = var_1799_dtype_0, x = var_1798)[name = tensor("cast_147")]; - tensor fill_mask_29 = reshape(shape = var_1800, x = var_1799)[name = tensor("fill_mask_29")]; + tensor var_1207 = equal(x = encoder__conv_positions, y = clip_28)[name = tensor("op_1207")]; + tensor var_1208_dtype_0 = const()[name = tensor("op_1208_dtype_0"), val = tensor("fp32")]; + tensor var_1209 = const()[name = tensor("op_1209"), val = tensor([1, 16, 1])]; + tensor var_1208 = cast(dtype = var_1208_dtype_0, x = var_1207)[name = tensor("cast_147")]; + tensor fill_mask_29 = reshape(shape = var_1209, x = var_1208)[name = tensor("fill_mask_29")]; tensor x_expanded_29_reps_0 = const()[name = tensor("x_expanded_29_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_29 = tile(reps = x_expanded_29_reps_0, x = var_1790)[name = tensor("x_expanded_29")]; - tensor var_1804 = sub(x = var_17, y = fill_mask_29)[name = tensor("op_1804")]; - tensor var_1805 = mul(x = input_241, y = var_1804)[name = tensor("op_1805")]; - tensor var_1806 = mul(x = x_expanded_29, y = fill_mask_29)[name = tensor("op_1806")]; - tensor filled_window_29 = add(x = var_1805, y = var_1806)[name = tensor("filled_window_29")]; - tensor var_1808 = greater_equal(x = clip_27, y = var_16)[name = tensor("op_1808")]; - tensor var_1809_dtype_0 = const()[name = tensor("op_1809_dtype_0"), val = tensor("fp32")]; - tensor var_1810 = const()[name = tensor("op_1810"), val = tensor([1, 1, 1])]; - tensor var_1809 = cast(dtype = var_1809_dtype_0, x = var_1808)[name = tensor("cast_146")]; - tensor use_roll_29 = reshape(shape = var_1810, x = var_1809)[name = tensor("use_roll_29")]; - tensor var_1812 = sub(x = var_17, y = use_roll_29)[name = tensor("op_1812")]; - tensor var_1813 = mul(x = filled_window_29, y = var_1812)[name = tensor("op_1813")]; - tensor var_1814 = mul(x = rolled_29, y = use_roll_29)[name = tensor("op_1814")]; - tensor input_251 = add(x = var_1813, y = var_1814)[name = tensor("input_251")]; - tensor var_1816 = const()[name = tensor("op_1816"), val = tensor(0x1p+0)]; - tensor var_1817 = add(x = clip_27, y = var_1816)[name = tensor("op_1817")]; - tensor clip_29 = clip(alpha = var_21, beta = var_16, x = var_1817)[name = tensor("clip_29")]; - tensor x_45_axes_0 = const()[name = tensor("x_45_axes_0"), val = tensor([-1])]; - tensor x_45 = layer_norm(axes = x_45_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_251)[name = tensor("x_45")]; - tensor input_253_perm_0 = const()[name = tensor("input_253_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_99_pad_type_0 = const()[name = tensor("inputs_99_pad_type_0"), val = tensor("valid")]; - tensor inputs_99_strides_0 = const()[name = tensor("inputs_99_strides_0"), val = tensor([1])]; - tensor inputs_99_pad_0 = const()[name = tensor("inputs_99_pad_0"), val = tensor([0, 0])]; - tensor inputs_99_dilations_0 = const()[name = tensor("inputs_99_dilations_0"), val = tensor([1])]; - tensor inputs_99_groups_0 = const()[name = tensor("inputs_99_groups_0"), val = tensor(1)]; - tensor input_253 = transpose(perm = input_253_perm_0, x = x_45)[name = tensor("transpose_34")]; - tensor inputs_99 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_99_dilations_0, groups = inputs_99_groups_0, pad = inputs_99_pad_0, pad_type = inputs_99_pad_type_0, strides = inputs_99_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_253)[name = tensor("inputs_99")]; - tensor var_1838_split_sizes_0 = const()[name = tensor("op_1838_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1838_axis_0 = const()[name = tensor("op_1838_axis_0"), val = tensor(1)]; - tensor var_1838_0, tensor var_1838_1 = split(axis = var_1838_axis_0, split_sizes = var_1838_split_sizes_0, x = inputs_99)[name = tensor("op_1838")]; - tensor var_1840 = sigmoid(x = var_1838_1)[name = tensor("op_1840")]; - tensor inputs_101 = mul(x = var_1838_0, y = var_1840)[name = tensor("inputs_101")]; - tensor outputs_aug_29_pad_type_0 = const()[name = tensor("outputs_aug_29_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_29_pad_0 = const()[name = tensor("outputs_aug_29_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_29_groups_0 = const()[name = tensor("outputs_aug_29_groups_0"), val = tensor(256)]; - tensor outputs_aug_29_strides_0 = const()[name = tensor("outputs_aug_29_strides_0"), val = tensor([1])]; - tensor outputs_aug_29_dilations_0 = const()[name = tensor("outputs_aug_29_dilations_0"), val = tensor([1])]; - tensor outputs_aug_29 = conv(dilations = outputs_aug_29_dilations_0, groups = outputs_aug_29_groups_0, pad = outputs_aug_29_pad_0, pad_type = outputs_aug_29_pad_type_0, strides = outputs_aug_29_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_101)[name = tensor("outputs_aug_29")]; - tensor input_255_begin_0 = const()[name = tensor("input_255_begin_0"), val = tensor([0, 0, 0])]; - tensor input_255_end_0 = const()[name = tensor("input_255_end_0"), val = tensor([1, 256, 16])]; - tensor input_255_end_mask_0 = const()[name = tensor("input_255_end_mask_0"), val = tensor([true, true, false])]; - tensor input_255 = slice_by_index(begin = input_255_begin_0, end = input_255_end_0, end_mask = input_255_end_mask_0, x = outputs_aug_29)[name = tensor("input_255")]; - tensor inputs_103 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_255)[name = tensor("inputs_103")]; - tensor input_257 = silu(x = inputs_103)[name = tensor("input_257")]; - tensor input_259_pad_type_0 = const()[name = tensor("input_259_pad_type_0"), val = tensor("valid")]; - tensor input_259_strides_0 = const()[name = tensor("input_259_strides_0"), val = tensor([1])]; - tensor input_259_pad_0 = const()[name = tensor("input_259_pad_0"), val = tensor([0, 0])]; - tensor input_259_dilations_0 = const()[name = tensor("input_259_dilations_0"), val = tensor([1])]; - tensor input_259_groups_0 = const()[name = tensor("input_259_groups_0"), val = tensor(1)]; - tensor input_259 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_259_dilations_0, groups = input_259_groups_0, pad = input_259_pad_0, pad_type = input_259_pad_type_0, strides = input_259_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_257)[name = tensor("input_259")]; - tensor conv_out_29_perm_0 = const()[name = tensor("conv_out_29_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1871_begin_0 = const()[name = tensor("op_1871_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1871_end_0 = const()[name = tensor("op_1871_end_0"), val = tensor([1, 16, 256])]; - tensor var_1871_end_mask_0 = const()[name = tensor("op_1871_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_29 = transpose(perm = conv_out_29_perm_0, x = input_259)[name = tensor("transpose_33")]; - tensor var_1871 = slice_by_index(begin = var_1871_begin_0, end = var_1871_end_0, end_mask = var_1871_end_mask_0, x = conv_out_29)[name = tensor("op_1871")]; - tensor var_1874_begin_0 = const()[name = tensor("op_1874_begin_0"), val = tensor([0, 3, 0])]; - tensor var_1874_end_0 = const()[name = tensor("op_1874_end_0"), val = tensor([1, 1, 256])]; - tensor var_1874_end_mask_0 = const()[name = tensor("op_1874_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1874 = slice_by_index(begin = var_1874_begin_0, end = var_1874_end_0, end_mask = var_1874_end_mask_0, x = x_39)[name = tensor("op_1874")]; - tensor var_1877_begin_0 = const()[name = tensor("op_1877_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1877_end_0 = const()[name = tensor("op_1877_end_0"), val = tensor([1, 16, 256])]; - tensor var_1877_end_mask_0 = const()[name = tensor("op_1877_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1877 = slice_by_index(begin = var_1877_begin_0, end = var_1877_end_0, end_mask = var_1877_end_mask_0, x = input_251)[name = tensor("op_1877")]; + tensor x_expanded_29 = tile(reps = x_expanded_29_reps_0, x = var_1199)[name = tensor("x_expanded_29")]; + tensor var_1213 = sub(x = var_17, y = fill_mask_29)[name = tensor("op_1213")]; + tensor var_1214 = mul(x = window_35, y = var_1213)[name = tensor("op_1214")]; + tensor var_1215 = mul(x = x_expanded_29, y = fill_mask_29)[name = tensor("op_1215")]; + tensor filled_window_29 = add(x = var_1214, y = var_1215)[name = tensor("filled_window_29")]; + tensor var_1217 = greater_equal(x = clip_27, y = var_16)[name = tensor("op_1217")]; + tensor var_1218_dtype_0 = const()[name = tensor("op_1218_dtype_0"), val = tensor("fp32")]; + tensor var_1219 = const()[name = tensor("op_1219"), val = tensor([1, 1, 1])]; + tensor var_1218 = cast(dtype = var_1218_dtype_0, x = var_1217)[name = tensor("cast_146")]; + tensor use_roll_29 = reshape(shape = var_1219, x = var_1218)[name = tensor("use_roll_29")]; + tensor var_1221 = sub(x = var_17, y = use_roll_29)[name = tensor("op_1221")]; + tensor var_1222 = mul(x = filled_window_29, y = var_1221)[name = tensor("op_1222")]; + tensor var_1223 = mul(x = rolled_29, y = use_roll_29)[name = tensor("op_1223")]; + tensor window_37 = add(x = var_1222, y = var_1223)[name = tensor("window_37")]; + tensor var_1225 = const()[name = tensor("op_1225"), val = tensor(0x1p+0)]; + tensor var_1226 = add(x = clip_27, y = var_1225)[name = tensor("op_1226")]; + tensor clip_29 = clip(alpha = var_21, beta = var_16, x = var_1226)[name = tensor("clip_29")]; + tensor var_1229_begin_0 = const()[name = tensor("op_1229_begin_0"), val = tensor([0, 3, 0])]; + tensor var_1229_end_0 = const()[name = tensor("op_1229_end_0"), val = tensor([1, 1, 256])]; + tensor var_1229_end_mask_0 = const()[name = tensor("op_1229_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1229 = slice_by_index(begin = var_1229_begin_0, end = var_1229_end_0, end_mask = var_1229_end_mask_0, x = x_21)[name = tensor("op_1229")]; + tensor var_1232_begin_0 = const()[name = tensor("op_1232_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1232_end_0 = const()[name = tensor("op_1232_end_0"), val = tensor([1, 16, 256])]; + tensor var_1232_end_mask_0 = const()[name = tensor("op_1232_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1232 = slice_by_index(begin = var_1232_begin_0, end = var_1232_end_0, end_mask = var_1232_end_mask_0, x = window_37)[name = tensor("op_1232")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1877, var_1874))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1232, var_1229))[name = tensor("rolled")]; tensor clip_30 = clip(alpha = var_21, beta = var_20, x = clip_29)[name = tensor("clip_30")]; - tensor var_1882 = equal(x = encoder__conv_positions, y = clip_30)[name = tensor("op_1882")]; - tensor var_1883_dtype_0 = const()[name = tensor("op_1883_dtype_0"), val = tensor("fp32")]; - tensor var_1884 = const()[name = tensor("op_1884"), val = tensor([1, 16, 1])]; - tensor var_1883 = cast(dtype = var_1883_dtype_0, x = var_1882)[name = tensor("cast_145")]; - tensor fill_mask = reshape(shape = var_1884, x = var_1883)[name = tensor("fill_mask")]; + tensor var_1237 = equal(x = encoder__conv_positions, y = clip_30)[name = tensor("op_1237")]; + tensor var_1238_dtype_0 = const()[name = tensor("op_1238_dtype_0"), val = tensor("fp32")]; + tensor var_1239 = const()[name = tensor("op_1239"), val = tensor([1, 16, 1])]; + tensor var_1238 = cast(dtype = var_1238_dtype_0, x = var_1237)[name = tensor("cast_145")]; + tensor fill_mask = reshape(shape = var_1239, x = var_1238)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1874)[name = tensor("x_expanded")]; - tensor var_1888 = sub(x = var_17, y = fill_mask)[name = tensor("op_1888")]; - tensor var_1889 = mul(x = input_251, y = var_1888)[name = tensor("op_1889")]; - tensor var_1890 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1890")]; - tensor filled_window = add(x = var_1889, y = var_1890)[name = tensor("filled_window")]; - tensor var_1892 = greater_equal(x = clip_29, y = var_16)[name = tensor("op_1892")]; - tensor var_1893_dtype_0 = const()[name = tensor("op_1893_dtype_0"), val = tensor("fp32")]; - tensor var_1894 = const()[name = tensor("op_1894"), val = tensor([1, 1, 1])]; - tensor var_1893 = cast(dtype = var_1893_dtype_0, x = var_1892)[name = tensor("cast_144")]; - tensor use_roll = reshape(shape = var_1894, x = var_1893)[name = tensor("use_roll")]; - tensor var_1896 = sub(x = var_17, y = use_roll)[name = tensor("op_1896")]; - tensor var_1897 = mul(x = filled_window, y = var_1896)[name = tensor("op_1897")]; - tensor var_1898 = mul(x = rolled, y = use_roll)[name = tensor("op_1898")]; - tensor input_261 = add(x = var_1897, y = var_1898)[name = tensor("input_261")]; - tensor var_1900 = const()[name = tensor("op_1900"), val = tensor(0x1p+0)]; - tensor var_1901 = add(x = clip_29, y = var_1900)[name = tensor("op_1901")]; - tensor clip_31 = clip(alpha = var_21, beta = var_16, x = var_1901)[name = tensor("clip_31")]; - tensor x_47_axes_0 = const()[name = tensor("x_47_axes_0"), val = tensor([-1])]; - tensor x_47 = layer_norm(axes = x_47_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_261)[name = tensor("x_47")]; - tensor input_263_perm_0 = const()[name = tensor("input_263_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_105_pad_type_0 = const()[name = tensor("inputs_105_pad_type_0"), val = tensor("valid")]; - tensor inputs_105_strides_0 = const()[name = tensor("inputs_105_strides_0"), val = tensor([1])]; - tensor inputs_105_pad_0 = const()[name = tensor("inputs_105_pad_0"), val = tensor([0, 0])]; - tensor inputs_105_dilations_0 = const()[name = tensor("inputs_105_dilations_0"), val = tensor([1])]; - tensor inputs_105_groups_0 = const()[name = tensor("inputs_105_groups_0"), val = tensor(1)]; - tensor input_263 = transpose(perm = input_263_perm_0, x = x_47)[name = tensor("transpose_32")]; - tensor inputs_105 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_105_dilations_0, groups = inputs_105_groups_0, pad = inputs_105_pad_0, pad_type = inputs_105_pad_type_0, strides = inputs_105_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_263)[name = tensor("inputs_105")]; - tensor var_1922_split_sizes_0 = const()[name = tensor("op_1922_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1922_axis_0 = const()[name = tensor("op_1922_axis_0"), val = tensor(1)]; - tensor var_1922_0, tensor var_1922_1 = split(axis = var_1922_axis_0, split_sizes = var_1922_split_sizes_0, x = inputs_105)[name = tensor("op_1922")]; - tensor var_1924 = sigmoid(x = var_1922_1)[name = tensor("op_1924")]; - tensor inputs_107 = mul(x = var_1922_0, y = var_1924)[name = tensor("inputs_107")]; + tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1229)[name = tensor("x_expanded")]; + tensor var_1243 = sub(x = var_17, y = fill_mask)[name = tensor("op_1243")]; + tensor var_1244 = mul(x = window_37, y = var_1243)[name = tensor("op_1244")]; + tensor var_1245 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1245")]; + tensor filled_window = add(x = var_1244, y = var_1245)[name = tensor("filled_window")]; + tensor var_1247 = greater_equal(x = clip_29, y = var_16)[name = tensor("op_1247")]; + tensor var_1248_dtype_0 = const()[name = tensor("op_1248_dtype_0"), val = tensor("fp32")]; + tensor var_1249 = const()[name = tensor("op_1249"), val = tensor([1, 1, 1])]; + tensor var_1248 = cast(dtype = var_1248_dtype_0, x = var_1247)[name = tensor("cast_144")]; + tensor use_roll = reshape(shape = var_1249, x = var_1248)[name = tensor("use_roll")]; + tensor var_1251 = sub(x = var_17, y = use_roll)[name = tensor("op_1251")]; + tensor var_1252 = mul(x = filled_window, y = var_1251)[name = tensor("op_1252")]; + tensor var_1253 = mul(x = rolled, y = use_roll)[name = tensor("op_1253")]; + tensor window = add(x = var_1252, y = var_1253)[name = tensor("window")]; + tensor var_1255 = const()[name = tensor("op_1255"), val = tensor(0x1p+0)]; + tensor var_1256 = add(x = clip_29, y = var_1255)[name = tensor("op_1256")]; + tensor clip_31 = clip(alpha = var_21, beta = var_16, x = var_1256)[name = tensor("clip_31")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_30, interleave = input_141_interleave_0, values = (window_33, window_35, window_37, window))[name = tensor("input_141")]; + tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; + tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; + tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_33_pad_type_0 = const()[name = tensor("inputs_33_pad_type_0"), val = tensor("valid")]; + tensor inputs_33_strides_0 = const()[name = tensor("inputs_33_strides_0"), val = tensor([1])]; + tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; + tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; + tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; + tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; + tensor var_1279_split_sizes_0 = const()[name = tensor("op_1279_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1279_axis_0 = const()[name = tensor("op_1279_axis_0"), val = tensor(1)]; + tensor var_1279_0, tensor var_1279_1 = split(axis = var_1279_axis_0, split_sizes = var_1279_split_sizes_0, x = inputs_33)[name = tensor("op_1279")]; + tensor var_1281 = sigmoid(x = var_1279_1)[name = tensor("op_1281")]; + tensor inputs_35 = mul(x = var_1279_0, y = var_1281)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; tensor outputs_aug_strides_0 = const()[name = tensor("outputs_aug_strides_0"), val = tensor([1])]; tensor outputs_aug_dilations_0 = const()[name = tensor("outputs_aug_dilations_0"), val = tensor([1])]; - tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_107)[name = tensor("outputs_aug")]; - tensor input_265_begin_0 = const()[name = tensor("input_265_begin_0"), val = tensor([0, 0, 0])]; - tensor input_265_end_0 = const()[name = tensor("input_265_end_0"), val = tensor([1, 256, 16])]; - tensor input_265_end_mask_0 = const()[name = tensor("input_265_end_mask_0"), val = tensor([true, true, false])]; - tensor input_265 = slice_by_index(begin = input_265_begin_0, end = input_265_end_0, end_mask = input_265_end_mask_0, x = outputs_aug)[name = tensor("input_265")]; - tensor inputs_109 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_265)[name = tensor("inputs_109")]; - tensor input_267 = silu(x = inputs_109)[name = tensor("input_267")]; - tensor input_269_pad_type_0 = const()[name = tensor("input_269_pad_type_0"), val = tensor("valid")]; - tensor input_269_strides_0 = const()[name = tensor("input_269_strides_0"), val = tensor([1])]; - tensor input_269_pad_0 = const()[name = tensor("input_269_pad_0"), val = tensor([0, 0])]; - tensor input_269_dilations_0 = const()[name = tensor("input_269_dilations_0"), val = tensor([1])]; - tensor input_269_groups_0 = const()[name = tensor("input_269_groups_0"), val = tensor(1)]; - tensor input_269 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_269_dilations_0, groups = input_269_groups_0, pad = input_269_pad_0, pad_type = input_269_pad_type_0, strides = input_269_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_267)[name = tensor("input_269")]; - tensor conv_out_31_perm_0 = const()[name = tensor("conv_out_31_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1955_begin_0 = const()[name = tensor("op_1955_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1955_end_0 = const()[name = tensor("op_1955_end_0"), val = tensor([1, 16, 256])]; - tensor var_1955_end_mask_0 = const()[name = tensor("op_1955_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_31 = transpose(perm = conv_out_31_perm_0, x = input_269)[name = tensor("transpose_31")]; - tensor var_1955 = slice_by_index(begin = var_1955_begin_0, end = var_1955_end_0, end_mask = var_1955_end_mask_0, x = conv_out_31)[name = tensor("op_1955")]; - tensor var_1958_interleave_0 = const()[name = tensor("op_1958_interleave_0"), val = tensor(false)]; - tensor var_1958 = concat(axis = var_33, interleave = var_1958_interleave_0, values = (var_1703, var_1787, var_1871, var_1955))[name = tensor("op_1958")]; - tensor input_271 = add(x = x_39, y = var_1958)[name = tensor("input_271")]; - tensor input_273_axes_0 = const()[name = tensor("input_273_axes_0"), val = tensor([-1])]; - tensor input_273 = layer_norm(axes = input_273_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_271)[name = tensor("input_273")]; - tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_273)[name = tensor("linear_35")]; - tensor input_275 = silu(x = inputs)[name = tensor("input_275")]; - tensor input_279 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_275)[name = tensor("linear_36")]; - tensor var_1981 = const()[name = tensor("op_1981"), val = tensor(0x1p-1)]; - tensor var_1982 = mul(x = input_279, y = var_1981)[name = tensor("op_1982")]; - tensor input_281 = add(x = var_1982, y = input_271)[name = tensor("input_281")]; - tensor x_49_axes_0 = const()[name = tensor("x_49_axes_0"), val = tensor([-1])]; - tensor x_49 = layer_norm(axes = x_49_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_281)[name = tensor("x_49")]; + tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_35)[name = tensor("outputs_aug")]; + tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; + tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([4, 256, 16])]; + tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; + tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug)[name = tensor("input_145")]; + tensor inputs_37 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_145)[name = tensor("inputs_37")]; + tensor input_147 = silu(x = inputs_37)[name = tensor("input_147")]; + tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; + tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; + tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; + tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; + tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; + tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; + tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1312_begin_0 = const()[name = tensor("op_1312_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1312_end_0 = const()[name = tensor("op_1312_end_0"), val = tensor([4, 16, 256])]; + tensor var_1312_end_mask_0 = const()[name = tensor("op_1312_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_1312 = slice_by_index(begin = var_1312_begin_0, end = var_1312_end_0, end_mask = var_1312_end_mask_0, x = conv_out_7)[name = tensor("op_1312")]; + tensor var_1314_perm_0 = const()[name = tensor("op_1314_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1314 = transpose(perm = var_1314_perm_0, x = var_1312)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_1314)[name = tensor("input_151")]; + tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; + tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; + tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; + tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; + tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; + tensor var_1337 = const()[name = tensor("op_1337"), val = tensor(0x1p-1)]; + tensor var_1338 = mul(x = input_159, y = var_1337)[name = tensor("op_1338")]; + tensor input_161 = add(x = var_1338, y = input_151)[name = tensor("input_161")]; + tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; + tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; tensor cat_interleave_0 = const()[name = tensor("cat_interleave_0"), val = tensor(false)]; - tensor x_ct = transpose(perm = x_ct_perm_0, x = x_49)[name = tensor("transpose_30")]; + tensor x_ct = transpose(perm = x_ct_perm_0, x = x_25)[name = tensor("transpose_30")]; tensor cat = concat(axis = var_27, interleave = cat_interleave_0, values = (cnn_window, x_ct))[name = tensor("cat")]; tensor conv_out_pad_type_0 = const()[name = tensor("conv_out_pad_type_0"), val = tensor("valid")]; tensor conv_out_strides_0 = const()[name = tensor("conv_out_strides_0"), val = tensor([1])]; @@ -1805,44 +1346,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_2000_begin_0 = const()[name = tensor("op_2000_begin_0"), val = tensor([0, 0, 4])]; - tensor var_2000_end_0 = const()[name = tensor("op_2000_end_0"), val = tensor([1, 256, 22])]; - tensor var_2000_end_mask_0 = const()[name = tensor("op_2000_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_2000_begin_0, end = var_2000_end_0, end_mask = var_2000_end_mask_0, x = cat)[name = tensor("op_2000")]; - tensor input_283_perm_0 = const()[name = tensor("input_283_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2002 = const()[name = tensor("op_2002"), val = tensor([-1])]; - tensor input_283 = transpose(perm = input_283_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_2003 = reduce_l2_norm(axes = var_2002, keep_dims = var_36, x = input_283)[name = tensor("op_2003")]; - tensor const_24 = const()[name = tensor("const_24"), val = tensor(0x1.fffffep+127)]; - tensor clip_32 = clip(alpha = var_12, beta = const_24, x = var_2003)[name = tensor("clip_32")]; - tensor emb = real_div(x = input_283, y = clip_32)[name = tensor("emb")]; - tensor var_2007_axis_0 = const()[name = tensor("op_2007_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_2007_axis_0, values = (var_214, var_678, var_1142, nkv_1))[name = tensor("op_2007")]; - tensor var_2009_axis_0 = const()[name = tensor("op_2009_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_2009_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_2009")]; - tensor var_2011_axis_0 = const()[name = tensor("op_2011_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_2011_axis_0, values = (input_51, input_121, input_191, input_261))[name = tensor("op_2011")]; - tensor var_2013_axis_0 = const()[name = tensor("op_2013_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_2013_axis_0, values = (clip_7, clip_15, clip_23, clip_31))[name = tensor("op_2013")]; - tensor var_2023 = const()[name = tensor("op_2023"), val = tensor(0x1.5798eep-27)]; - tensor var_2029 = const()[name = tensor("op_2029"), val = tensor(0x1.4f8b58p-17)]; - tensor var_2031 = const()[name = tensor("op_2031"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_2032 = const()[name = tensor("op_2032"), val = tensor(true)]; - tensor var_2034 = const()[name = tensor("op_2034"), val = tensor(0x1p+0)]; - tensor var_2038 = const()[name = tensor("op_2038"), val = tensor(-1)]; - tensor var_2043 = const()[name = tensor("op_2043"), val = tensor(0)]; + tensor var_1356_begin_0 = const()[name = tensor("op_1356_begin_0"), val = tensor([0, 0, 4])]; + tensor var_1356_end_0 = const()[name = tensor("op_1356_end_0"), val = tensor([1, 256, 22])]; + tensor var_1356_end_mask_0 = const()[name = tensor("op_1356_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_1356_begin_0, end = var_1356_end_0, end_mask = var_1356_end_mask_0, x = cat)[name = tensor("op_1356")]; + tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1358 = const()[name = tensor("op_1358"), val = tensor([-1])]; + tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; + tensor var_1359 = reduce_l2_norm(axes = var_1358, keep_dims = var_36, x = input_163)[name = tensor("op_1359")]; + tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; + tensor clip_32 = clip(alpha = var_12, beta = const_12, x = var_1359)[name = tensor("clip_32")]; + tensor emb = real_div(x = input_163, y = clip_32)[name = tensor("emb")]; + tensor var_1363_axis_0 = const()[name = tensor("op_1363_axis_0"), val = tensor(0)]; + tensor enc_kv_new = stack(axis = var_1363_axis_0, values = (var_214, var_517, var_820, nkv_1))[name = tensor("op_1363")]; + tensor var_1365_axis_0 = const()[name = tensor("op_1365_axis_0"), val = tensor(0)]; + tensor enc_scale_new = stack(axis = var_1365_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1365")]; + tensor var_1367_axis_0 = const()[name = tensor("op_1367_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1367_axis_0, values = (window_9, window_19, window_29, window))[name = tensor("op_1367")]; + tensor var_1369_axis_0 = const()[name = tensor("op_1369_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1369_axis_0, values = (clip_7, clip_15, clip_23, clip_31))[name = tensor("op_1369")]; + tensor var_1379 = const()[name = tensor("op_1379"), val = tensor(0x1.5798eep-27)]; + tensor var_1384 = const()[name = tensor("op_1384"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1386 = const()[name = tensor("op_1386"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1387 = const()[name = tensor("op_1387"), val = tensor(true)]; + tensor var_1389 = const()[name = tensor("op_1389"), val = tensor(0x1p+0)]; + tensor var_1393 = const()[name = tensor("op_1393"), val = tensor(-1)]; + tensor var_1399 = const()[name = tensor("op_1399"), val = tensor(0)]; tensor pos = const()[name = tensor("pos"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395712)))]; - tensor var_2105_axes_0 = const()[name = tensor("op_2105_axes_0"), val = tensor([2])]; - tensor var_2105 = expand_dims(axes = var_2105_axes_0, x = emb)[name = tensor("op_2105")]; - tensor var_2106 = const()[name = tensor("op_2106"), val = tensor([1, 1, 9, 1])]; - tensor emb_exp = tile(reps = var_2106, x = var_2105)[name = tensor("emb_exp")]; - tensor input_285_interleave_0 = const()[name = tensor("input_285_interleave_0"), val = tensor(false)]; - tensor input_285 = concat(axis = var_2038, interleave = input_285_interleave_0, values = (emb_exp, pos))[name = tensor("input_285")]; - tensor x_51 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_285)[name = tensor("linear_37")]; - tensor var_2113_perm_0 = const()[name = tensor("op_2113_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2117 = const()[name = tensor("op_2117"), val = tensor([9, 4, 256])]; - tensor var_2113 = transpose(perm = var_2113_perm_0, x = x_51)[name = tensor("transpose_28")]; - tensor x_53 = reshape(shape = var_2117, x = var_2113)[name = tensor("x_53")]; + tensor var_1461_axes_0 = const()[name = tensor("op_1461_axes_0"), val = tensor([2])]; + tensor var_1461 = expand_dims(axes = var_1461_axes_0, x = emb)[name = tensor("op_1461")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 9, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1461)[name = tensor("emb_exp")]; + tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; + tensor input_165 = concat(axis = var_1393, interleave = input_165_interleave_0, values = (emb_exp, pos))[name = tensor("input_165")]; + tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; + tensor var_1469_perm_0 = const()[name = tensor("op_1469_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1473 = const()[name = tensor("op_1473"), val = tensor([9, 4, 256])]; + tensor var_1469 = transpose(perm = var_1469_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1473, x = var_1469)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 9, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1853,132 +1394,132 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_2125 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_53)[name = tensor("linear_38")]; - tensor var_2126 = const()[name = tensor("op_2126"), val = tensor([9, 4, 4, 64])]; - tensor var_2127 = reshape(shape = var_2126, x = var_2125)[name = tensor("op_2127")]; + tensor var_1481 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1482 = const()[name = tensor("op_1482"), val = tensor([9, 4, 4, 64])]; + tensor var_1483 = reshape(shape = var_1482, x = var_1481)[name = tensor("op_1483")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2131 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_53)[name = tensor("linear_39")]; - tensor var_2132 = const()[name = tensor("op_2132"), val = tensor(0x1p-3)]; - tensor var_2133 = mul(x = var_2131, y = var_2132)[name = tensor("op_2133")]; - tensor var_2134 = const()[name = tensor("op_2134"), val = tensor([9, 4, 4, 64])]; - tensor var_2135 = reshape(shape = var_2134, x = var_2133)[name = tensor("op_2135")]; + tensor var_1487 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1488 = const()[name = tensor("op_1488"), val = tensor(0x1p-3)]; + tensor var_1489 = mul(x = var_1487, y = var_1488)[name = tensor("op_1489")]; + tensor var_1490 = const()[name = tensor("op_1490"), val = tensor([9, 4, 4, 64])]; + tensor var_1491 = reshape(shape = var_1490, x = var_1489)[name = tensor("op_1491")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2139 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_53)[name = tensor("linear_40")]; - tensor var_2140 = const()[name = tensor("op_2140"), val = tensor([9, 4, 4, 64])]; - tensor var_2141 = reshape(shape = var_2140, x = var_2139)[name = tensor("op_2141")]; + tensor var_1495 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1496 = const()[name = tensor("op_1496"), val = tensor([9, 4, 4, 64])]; + tensor var_1497 = reshape(shape = var_1496, x = var_1495)[name = tensor("op_1497")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_289 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_53)[name = tensor("linear_41")]; + tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_2043, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1399, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; - tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; - tensor clip_33 = clip(alpha = var_2034, beta = const_32, x = s_t_9)[name = tensor("clip_33")]; + tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; + tensor clip_33 = clip(alpha = var_1389, beta = const_20, x = s_t_9)[name = tensor("clip_33")]; tensor sqrt_s_t_9 = sqrt(x = clip_33)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_2135)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_2127)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1491)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1483)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_2153 = const()[name = tensor("op_2153"), val = tensor([1, 4])]; - tensor var_2154 = reshape(shape = var_2153, x = valid_mask)[name = tensor("op_2154")]; - tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_2154)[name = tensor("causal_with_valid_1")]; - tensor var_2156 = const()[name = tensor("op_2156"), val = tensor([4, 1])]; - tensor var_2157 = reshape(shape = var_2156, x = sqrt_s_t_9)[name = tensor("op_2157")]; - tensor M_9 = real_div(x = causal_with_valid_1, y = var_2157)[name = tensor("M_9")]; - tensor var_2159 = mul(x = qk_9, y = M_9)[name = tensor("op_2159")]; + tensor var_1509 = const()[name = tensor("op_1509"), val = tensor([1, 4])]; + tensor var_1510 = reshape(shape = var_1509, x = valid_mask)[name = tensor("op_1510")]; + tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1510)[name = tensor("causal_with_valid_1")]; + tensor var_1512 = const()[name = tensor("op_1512"), val = tensor([4, 1])]; + tensor var_1513 = reshape(shape = var_1512, x = sqrt_s_t_9)[name = tensor("op_1513")]; + tensor M_9 = real_div(x = causal_with_valid_1, y = var_1513)[name = tensor("M_9")]; + tensor var_1515 = mul(x = qk_9, y = M_9)[name = tensor("op_1515")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_2141)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_2159, y = v_9)[name = tensor("inner_9")]; - tensor var_2161_transpose_x_0 = const()[name = tensor("op_2161_transpose_x_0"), val = tensor(false)]; - tensor var_2161_transpose_y_0 = const()[name = tensor("op_2161_transpose_y_0"), val = tensor(false)]; - tensor var_2161 = matmul(transpose_x = var_2161_transpose_x_0, transpose_y = var_2161_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_2161")]; - tensor var_2162 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_2162")]; - tensor var_2163 = const()[name = tensor("op_2163"), val = tensor([1, 1, 4, 1])]; - tensor var_2164 = reshape(shape = var_2163, x = var_2162)[name = tensor("op_2164")]; - tensor cross_9 = mul(x = var_2161, y = var_2164)[name = tensor("cross_9")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1497)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1515, y = v_9)[name = tensor("inner_9")]; + tensor var_1517_transpose_x_0 = const()[name = tensor("op_1517_transpose_x_0"), val = tensor(false)]; + tensor var_1517_transpose_y_0 = const()[name = tensor("op_1517_transpose_y_0"), val = tensor(false)]; + tensor var_1517 = matmul(transpose_x = var_1517_transpose_x_0, transpose_y = var_1517_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1517")]; + tensor var_1518 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1518")]; + tensor var_1519 = const()[name = tensor("op_1519"), val = tensor([1, 1, 4, 1])]; + tensor var_1520 = reshape(shape = var_1519, x = var_1518)[name = tensor("op_1520")]; + tensor cross_9 = mul(x = var_1517, y = var_1520)[name = tensor("cross_9")]; tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; - tensor var_2167 = const()[name = tensor("op_2167"), val = tensor([1, 1, 4, 1])]; - tensor var_2168 = reshape(shape = var_2167, x = valid_mask)[name = tensor("op_2168")]; - tensor v_masked_1 = mul(x = v_9, y = var_2168)[name = tensor("v_masked_1")]; - tensor var_2170 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_2170")]; - tensor var_2172_transpose_x_1 = const()[name = tensor("op_2172_transpose_x_1"), val = tensor(true)]; - tensor var_2172_transpose_y_1 = const()[name = tensor("op_2172_transpose_y_1"), val = tensor(false)]; - tensor var_2172 = matmul(transpose_x = var_2172_transpose_x_1, transpose_y = var_2172_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_2172")]; - tensor new_kv_unnorm_9 = add(x = var_2170, y = var_2172)[name = tensor("new_kv_unnorm_9")]; - tensor var_2174_keep_dims_0 = const()[name = tensor("op_2174_keep_dims_0"), val = tensor(false)]; - tensor var_2174 = reduce_sum(keep_dims = var_2174_keep_dims_0, x = valid_mask)[name = tensor("op_2174")]; - tensor var_2175 = const()[name = tensor("op_2175"), val = tensor([1])]; - tensor var_2176 = reshape(shape = var_2175, x = var_2174)[name = tensor("op_2176")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_2176)[name = tensor("new_scale_9")]; - tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; - tensor clip_34 = clip(alpha = var_2034, beta = const_33, x = new_scale_9)[name = tensor("clip_34")]; + tensor var_1523 = const()[name = tensor("op_1523"), val = tensor([1, 1, 4, 1])]; + tensor var_1524 = reshape(shape = var_1523, x = valid_mask)[name = tensor("op_1524")]; + tensor v_masked_1 = mul(x = v_9, y = var_1524)[name = tensor("v_masked_1")]; + tensor var_1526 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1526")]; + tensor var_1528_transpose_x_1 = const()[name = tensor("op_1528_transpose_x_1"), val = tensor(true)]; + tensor var_1528_transpose_y_1 = const()[name = tensor("op_1528_transpose_y_1"), val = tensor(false)]; + tensor var_1528 = matmul(transpose_x = var_1528_transpose_x_1, transpose_y = var_1528_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1528")]; + tensor new_kv_unnorm_9 = add(x = var_1526, y = var_1528)[name = tensor("new_kv_unnorm_9")]; + tensor var_1530_keep_dims_0 = const()[name = tensor("op_1530_keep_dims_0"), val = tensor(false)]; + tensor var_1530 = reduce_sum(keep_dims = var_1530_keep_dims_0, x = valid_mask)[name = tensor("op_1530")]; + tensor var_1531 = const()[name = tensor("op_1531"), val = tensor([1])]; + tensor var_1532 = reshape(shape = var_1531, x = var_1530)[name = tensor("op_1532")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1532)[name = tensor("new_scale_9")]; + tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; + tensor clip_34 = clip(alpha = var_1389, beta = const_21, x = new_scale_9)[name = tensor("clip_34")]; tensor sqrt_new_scale_1 = sqrt(x = clip_34)[name = tensor("sqrt_new_scale_1")]; - tensor var_2180 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_2180")]; - tensor var_2181_perm_0 = const()[name = tensor("op_2181_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1536 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1536")]; + tensor var_1537_perm_0 = const()[name = tensor("op_1537_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_2181 = transpose(perm = var_2181_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_2031, x = var_2181)[name = tensor("out_27")]; - tensor var_2185 = const()[name = tensor("op_2185"), val = tensor([9, 4, 256])]; - tensor out_29 = reshape(shape = var_2185, x = out_27)[name = tensor("out_29")]; - tensor var_2187 = silu(x = input_289)[name = tensor("op_2187")]; - tensor input_291 = mul(x = var_2187, y = out_29)[name = tensor("input_291")]; - tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_291)[name = tensor("linear_42")]; - tensor input_293 = add(x = x_53, y = ret_out_9)[name = tensor("input_293")]; + tensor var_1537 = transpose(perm = var_1537_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1386, x = var_1537)[name = tensor("out_27")]; + tensor var_1541 = const()[name = tensor("op_1541"), val = tensor([9, 4, 256])]; + tensor out_29 = reshape(shape = var_1541, x = out_27)[name = tensor("out_29")]; + tensor var_1543 = silu(x = input_169)[name = tensor("op_1543")]; + tensor input_171 = mul(x = var_1543, y = out_29)[name = tensor("input_171")]; + tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; + tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_2029, gamma = decoder_norm11_0_weight, x = input_293)[name = tensor("xt_1")]; - tensor var_2197 = const()[name = tensor("op_2197"), val = tensor([1, 9, 4, 256])]; - tensor var_2198 = reshape(shape = var_2197, x = xt_1)[name = tensor("op_2198")]; - tensor var_2199_perm_0 = const()[name = tensor("op_2199_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2202 = const()[name = tensor("op_2202"), val = tensor([4, 9, 256])]; - tensor var_2199 = transpose(perm = var_2199_perm_0, x = var_2198)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_2202, x = var_2199)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1384, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1553 = const()[name = tensor("op_1553"), val = tensor([1, 9, 4, 256])]; + tensor var_1554 = reshape(shape = var_1553, x = xt_1)[name = tensor("op_1554")]; + tensor var_1555_perm_0 = const()[name = tensor("op_1555_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1558 = const()[name = tensor("op_1558"), val = tensor([4, 9, 256])]; + tensor var_1555 = transpose(perm = var_1555_perm_0, x = var_1554)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1558, x = var_1555)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_2225 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_16 = const()[name = tensor("concat_16"), val = tensor([9, 4, 3, 256])]; - tensor var_2227 = reshape(shape = concat_16, x = var_2225)[name = tensor("op_2227")]; - tensor var_2228_axes_0 = const()[name = tensor("op_2228_axes_0"), val = tensor([0])]; - tensor var_2228 = expand_dims(axes = var_2228_axes_0, x = var_2227)[name = tensor("op_2228")]; - tensor var_2229_perm_0 = const()[name = tensor("op_2229_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2230_axes_0 = const()[name = tensor("op_2230_axes_0"), val = tensor([-2])]; - tensor var_2229 = transpose(perm = var_2229_perm_0, x = var_2228)[name = tensor("transpose_21")]; - tensor var_2230 = squeeze(axes = var_2230_axes_0, x = var_2229)[name = tensor("op_2230")]; + tensor var_1581 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_17 = const()[name = tensor("concat_17"), val = tensor([9, 4, 3, 256])]; + tensor var_1583 = reshape(shape = concat_17, x = var_1581)[name = tensor("op_1583")]; + tensor var_1584_axes_0 = const()[name = tensor("op_1584_axes_0"), val = tensor([0])]; + tensor var_1584 = expand_dims(axes = var_1584_axes_0, x = var_1583)[name = tensor("op_1584")]; + tensor var_1585_perm_0 = const()[name = tensor("op_1585_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1586_axes_0 = const()[name = tensor("op_1586_axes_0"), val = tensor([-2])]; + tensor var_1585 = transpose(perm = var_1585_perm_0, x = var_1584)[name = tensor("transpose_21")]; + tensor var_1586 = squeeze(axes = var_1586_axes_0, x = var_1585)[name = tensor("op_1586")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 9, 4, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_2230)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1586)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 9, 4, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_2230)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1586)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 9, 4, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_2230)[name = tensor("v_11")]; - tensor var_2238 = const()[name = tensor("op_2238"), val = tensor([9, 16, 64])]; - tensor var_2239 = reshape(shape = var_2238, x = q_11)[name = tensor("op_2239")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1586)[name = tensor("v_11")]; + tensor var_1594 = const()[name = tensor("op_1594"), val = tensor([9, 16, 64])]; + tensor var_1595 = reshape(shape = var_1594, x = q_11)[name = tensor("op_1595")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2245 = const()[name = tensor("op_2245"), val = tensor([9, 16, 64])]; - tensor var_2246 = reshape(shape = var_2245, x = k_11)[name = tensor("op_2246")]; + tensor var_1601 = const()[name = tensor("op_1601"), val = tensor([9, 16, 64])]; + tensor var_1602 = reshape(shape = var_1601, x = k_11)[name = tensor("op_1602")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2252 = const()[name = tensor("op_2252"), val = tensor([9, 16, 64])]; - tensor var_2253 = reshape(shape = var_2252, x = v_11)[name = tensor("op_2253")]; + tensor var_1608 = const()[name = tensor("op_1608"), val = tensor([9, 16, 64])]; + tensor var_1609 = reshape(shape = var_1608, x = v_11)[name = tensor("op_1609")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2256 = const()[name = tensor("op_2256"), val = tensor([4, 4, 9, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_2239)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_2256, x = q_13)[name = tensor("q_15")]; - tensor var_2258 = const()[name = tensor("op_2258"), val = tensor([4, 4, 9, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_2246)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_2258, x = k_13)[name = tensor("k_15")]; - tensor var_2260 = const()[name = tensor("op_2260"), val = tensor([4, 4, 9, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_2253)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_2260, x = v_13)[name = tensor("v_15")]; + tensor var_1612 = const()[name = tensor("op_1612"), val = tensor([4, 4, 9, 64])]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1595)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1612, x = q_13)[name = tensor("q_15")]; + tensor var_1614 = const()[name = tensor("op_1614"), val = tensor([4, 4, 9, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1602)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1614, x = k_13)[name = tensor("k_15")]; + tensor var_1616 = const()[name = tensor("op_1616"), val = tensor([4, 4, 9, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1609)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1616, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -1989,30 +1530,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_2263 = const()[name = tensor("op_2263"), val = tensor([2, 0, 1, 3])]; - tensor var_2268 = const()[name = tensor("op_2268"), val = tensor([36, 256])]; - tensor var_2264 = transpose(perm = var_2263, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_2268, x = var_2264)[name = tensor("attn_output_3")]; + tensor var_1619 = const()[name = tensor("op_1619"), val = tensor([2, 0, 1, 3])]; + tensor var_1624 = const()[name = tensor("op_1624"), val = tensor([36, 256])]; + tensor var_1620 = transpose(perm = var_1619, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1624, x = var_1620)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_2272 = const()[name = tensor("op_2272"), val = tensor([9, 4, 256])]; - tensor attn_output_7 = reshape(shape = var_2272, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1628 = const()[name = tensor("op_1628"), val = tensor([9, 4, 256])]; + tensor attn_output_7 = reshape(shape = var_1628, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; - tensor input_295 = add(x = query_1, y = sa2_out_1)[name = tensor("input_295")]; - tensor input_297_axes_0 = const()[name = tensor("input_297_axes_0"), val = tensor([-1])]; - tensor input_297 = layer_norm(axes = input_297_axes_0, beta = decoder_norm21_0_bias, epsilon = var_2029, gamma = decoder_norm21_0_weight, x = input_295)[name = tensor("input_297")]; - tensor input_299 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_297)[name = tensor("linear_45")]; - tensor input_301 = relu(x = input_299)[name = tensor("input_301")]; - tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_301)[name = tensor("linear_46")]; - tensor input_303 = add(x = input_297, y = ffn_out_1)[name = tensor("input_303")]; + tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; + tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1384, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; + tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; + tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; + tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_2029, gamma = decoder_norm22_0_weight, x = input_303)[name = tensor("xt_3")]; - tensor var_2292 = const()[name = tensor("op_2292"), val = tensor([1, 4, 9, 256])]; - tensor x_55 = reshape(shape = var_2292, x = xt_3)[name = tensor("x_55")]; - tensor var_2294_perm_0 = const()[name = tensor("op_2294_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2298 = const()[name = tensor("op_2298"), val = tensor([9, 4, 256])]; - tensor var_2294 = transpose(perm = var_2294_perm_0, x = x_55)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_2298, x = var_2294)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1384, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1648 = const()[name = tensor("op_1648"), val = tensor([1, 4, 9, 256])]; + tensor x_31 = reshape(shape = var_1648, x = xt_3)[name = tensor("x_31")]; + tensor var_1650_perm_0 = const()[name = tensor("op_1650_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1654 = const()[name = tensor("op_1654"), val = tensor([9, 4, 256])]; + tensor var_1650 = transpose(perm = var_1650_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1654, x = var_1650)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 9, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -2023,120 +1564,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_2306 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_2307 = const()[name = tensor("op_2307"), val = tensor([9, 4, 4, 64])]; - tensor var_2308 = reshape(shape = var_2307, x = var_2306)[name = tensor("op_2308")]; + tensor var_1662 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1663 = const()[name = tensor("op_1663"), val = tensor([9, 4, 4, 64])]; + tensor var_1664 = reshape(shape = var_1663, x = var_1662)[name = tensor("op_1664")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2312 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_2313 = const()[name = tensor("op_2313"), val = tensor(0x1p-3)]; - tensor var_2314 = mul(x = var_2312, y = var_2313)[name = tensor("op_2314")]; - tensor var_2315 = const()[name = tensor("op_2315"), val = tensor([9, 4, 4, 64])]; - tensor var_2316 = reshape(shape = var_2315, x = var_2314)[name = tensor("op_2316")]; + tensor var_1668 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1669 = const()[name = tensor("op_1669"), val = tensor(0x1p-3)]; + tensor var_1670 = mul(x = var_1668, y = var_1669)[name = tensor("op_1670")]; + tensor var_1671 = const()[name = tensor("op_1671"), val = tensor([9, 4, 4, 64])]; + tensor var_1672 = reshape(shape = var_1671, x = var_1670)[name = tensor("op_1672")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2320 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_2321 = const()[name = tensor("op_2321"), val = tensor([9, 4, 4, 64])]; - tensor var_2322 = reshape(shape = var_2321, x = var_2320)[name = tensor("op_2322")]; + tensor var_1676 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1677 = const()[name = tensor("op_1677"), val = tensor([9, 4, 4, 64])]; + tensor var_1678 = reshape(shape = var_1677, x = var_1676)[name = tensor("op_1678")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_307 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; + tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; - tensor const_44 = const()[name = tensor("const_44"), val = tensor(0x1.fffffep+127)]; - tensor clip_35 = clip(alpha = var_2034, beta = const_44, x = s_t)[name = tensor("clip_35")]; + tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; + tensor clip_35 = clip(alpha = var_1389, beta = const_32, x = s_t)[name = tensor("clip_35")]; tensor sqrt_s_t = sqrt(x = clip_35)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_2316)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_2308)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1672)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1664)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_2337 = const()[name = tensor("op_2337"), val = tensor([4, 1])]; - tensor var_2338 = reshape(shape = var_2337, x = sqrt_s_t)[name = tensor("op_2338")]; - tensor M = real_div(x = causal_with_valid_1, y = var_2338)[name = tensor("M")]; - tensor var_2340 = mul(x = qk, y = M)[name = tensor("op_2340")]; + tensor var_1693 = const()[name = tensor("op_1693"), val = tensor([4, 1])]; + tensor var_1694 = reshape(shape = var_1693, x = sqrt_s_t)[name = tensor("op_1694")]; + tensor M = real_div(x = causal_with_valid_1, y = var_1694)[name = tensor("M")]; + tensor var_1696 = mul(x = qk, y = M)[name = tensor("op_1696")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_2322)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_2340, y = v_17)[name = tensor("inner")]; - tensor var_2342_transpose_x_0 = const()[name = tensor("op_2342_transpose_x_0"), val = tensor(false)]; - tensor var_2342_transpose_y_0 = const()[name = tensor("op_2342_transpose_y_0"), val = tensor(false)]; - tensor var_2342 = matmul(transpose_x = var_2342_transpose_x_0, transpose_y = var_2342_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_2342")]; - tensor var_2343 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_2343")]; - tensor var_2344 = const()[name = tensor("op_2344"), val = tensor([1, 1, 4, 1])]; - tensor var_2345 = reshape(shape = var_2344, x = var_2343)[name = tensor("op_2345")]; - tensor cross = mul(x = var_2342, y = var_2345)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1678)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1696, y = v_17)[name = tensor("inner")]; + tensor var_1698_transpose_x_0 = const()[name = tensor("op_1698_transpose_x_0"), val = tensor(false)]; + tensor var_1698_transpose_y_0 = const()[name = tensor("op_1698_transpose_y_0"), val = tensor(false)]; + tensor var_1698 = matmul(transpose_x = var_1698_transpose_x_0, transpose_y = var_1698_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1698")]; + tensor var_1699 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1699")]; + tensor var_1700 = const()[name = tensor("op_1700"), val = tensor([1, 1, 4, 1])]; + tensor var_1701 = reshape(shape = var_1700, x = var_1699)[name = tensor("op_1701")]; + tensor cross = mul(x = var_1698, y = var_1701)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_2168)[name = tensor("v_masked")]; - tensor var_2351 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_2351")]; - tensor var_2353_transpose_x_1 = const()[name = tensor("op_2353_transpose_x_1"), val = tensor(true)]; - tensor var_2353_transpose_y_1 = const()[name = tensor("op_2353_transpose_y_1"), val = tensor(false)]; - tensor var_2353 = matmul(transpose_x = var_2353_transpose_x_1, transpose_y = var_2353_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_2353")]; - tensor new_kv_unnorm = add(x = var_2351, y = var_2353)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_2176)[name = tensor("new_scale")]; - tensor const_45 = const()[name = tensor("const_45"), val = tensor(0x1.fffffep+127)]; - tensor clip_36 = clip(alpha = var_2034, beta = const_45, x = new_scale)[name = tensor("clip_36")]; + tensor v_masked = mul(x = v_17, y = var_1524)[name = tensor("v_masked")]; + tensor var_1707 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1707")]; + tensor var_1709_transpose_x_1 = const()[name = tensor("op_1709_transpose_x_1"), val = tensor(true)]; + tensor var_1709_transpose_y_1 = const()[name = tensor("op_1709_transpose_y_1"), val = tensor(false)]; + tensor var_1709 = matmul(transpose_x = var_1709_transpose_x_1, transpose_y = var_1709_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1709")]; + tensor new_kv_unnorm = add(x = var_1707, y = var_1709)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1532)[name = tensor("new_scale")]; + tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; + tensor clip_36 = clip(alpha = var_1389, beta = const_33, x = new_scale)[name = tensor("clip_36")]; tensor sqrt_new_scale = sqrt(x = clip_36)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_2362_perm_0 = const()[name = tensor("op_2362_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1718_perm_0 = const()[name = tensor("op_1718_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_2362 = transpose(perm = var_2362_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_2031, x = var_2362)[name = tensor("out_33")]; - tensor var_2366 = const()[name = tensor("op_2366"), val = tensor([9, 4, 256])]; - tensor out = reshape(shape = var_2366, x = out_33)[name = tensor("out")]; - tensor var_2368 = silu(x = input_307)[name = tensor("op_2368")]; - tensor input_309 = mul(x = var_2368, y = out)[name = tensor("input_309")]; - tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_309)[name = tensor("linear_51")]; - tensor input_311 = add(x = x, y = ret_out)[name = tensor("input_311")]; + tensor var_1718 = transpose(perm = var_1718_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1386, x = var_1718)[name = tensor("out_33")]; + tensor var_1722 = const()[name = tensor("op_1722"), val = tensor([9, 4, 256])]; + tensor out = reshape(shape = var_1722, x = out_33)[name = tensor("out")]; + tensor var_1724 = silu(x = input_187)[name = tensor("op_1724")]; + tensor input_189 = mul(x = var_1724, y = out)[name = tensor("input_189")]; + tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; + tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_2029, gamma = decoder_norm11_1_weight, x = input_311)[name = tensor("xt_5")]; - tensor var_2378 = const()[name = tensor("op_2378"), val = tensor([1, 9, 4, 256])]; - tensor var_2379 = reshape(shape = var_2378, x = xt_5)[name = tensor("op_2379")]; - tensor var_2380_perm_0 = const()[name = tensor("op_2380_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2383 = const()[name = tensor("op_2383"), val = tensor([4, 9, 256])]; - tensor var_2380 = transpose(perm = var_2380_perm_0, x = var_2379)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_2383, x = var_2380)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1384, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1734 = const()[name = tensor("op_1734"), val = tensor([1, 9, 4, 256])]; + tensor var_1735 = reshape(shape = var_1734, x = xt_5)[name = tensor("op_1735")]; + tensor var_1736_perm_0 = const()[name = tensor("op_1736_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1739 = const()[name = tensor("op_1739"), val = tensor([4, 9, 256])]; + tensor var_1736 = transpose(perm = var_1736_perm_0, x = var_1735)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1739, x = var_1736)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_2406 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_17 = const()[name = tensor("concat_17"), val = tensor([9, 4, 3, 256])]; - tensor var_2408 = reshape(shape = concat_17, x = var_2406)[name = tensor("op_2408")]; - tensor var_2409_axes_0 = const()[name = tensor("op_2409_axes_0"), val = tensor([0])]; - tensor var_2409 = expand_dims(axes = var_2409_axes_0, x = var_2408)[name = tensor("op_2409")]; - tensor var_2410_perm_0 = const()[name = tensor("op_2410_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2411_axes_0 = const()[name = tensor("op_2411_axes_0"), val = tensor([-2])]; - tensor var_2410 = transpose(perm = var_2410_perm_0, x = var_2409)[name = tensor("transpose_8")]; - tensor var_2411 = squeeze(axes = var_2411_axes_0, x = var_2410)[name = tensor("op_2411")]; + tensor var_1762 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_18 = const()[name = tensor("concat_18"), val = tensor([9, 4, 3, 256])]; + tensor var_1764 = reshape(shape = concat_18, x = var_1762)[name = tensor("op_1764")]; + tensor var_1765_axes_0 = const()[name = tensor("op_1765_axes_0"), val = tensor([0])]; + tensor var_1765 = expand_dims(axes = var_1765_axes_0, x = var_1764)[name = tensor("op_1765")]; + tensor var_1766_perm_0 = const()[name = tensor("op_1766_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1767_axes_0 = const()[name = tensor("op_1767_axes_0"), val = tensor([-2])]; + tensor var_1766 = transpose(perm = var_1766_perm_0, x = var_1765)[name = tensor("transpose_8")]; + tensor var_1767 = squeeze(axes = var_1767_axes_0, x = var_1766)[name = tensor("op_1767")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 9, 4, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_2411)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1767)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 9, 4, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_2411)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1767)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 9, 4, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_2411)[name = tensor("v_19")]; - tensor var_2419 = const()[name = tensor("op_2419"), val = tensor([9, 16, 64])]; - tensor var_2420 = reshape(shape = var_2419, x = q_19)[name = tensor("op_2420")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1767)[name = tensor("v_19")]; + tensor var_1775 = const()[name = tensor("op_1775"), val = tensor([9, 16, 64])]; + tensor var_1776 = reshape(shape = var_1775, x = q_19)[name = tensor("op_1776")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2426 = const()[name = tensor("op_2426"), val = tensor([9, 16, 64])]; - tensor var_2427 = reshape(shape = var_2426, x = k_19)[name = tensor("op_2427")]; + tensor var_1782 = const()[name = tensor("op_1782"), val = tensor([9, 16, 64])]; + tensor var_1783 = reshape(shape = var_1782, x = k_19)[name = tensor("op_1783")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2433 = const()[name = tensor("op_2433"), val = tensor([9, 16, 64])]; - tensor var_2434 = reshape(shape = var_2433, x = v_19)[name = tensor("op_2434")]; + tensor var_1789 = const()[name = tensor("op_1789"), val = tensor([9, 16, 64])]; + tensor var_1790 = reshape(shape = var_1789, x = v_19)[name = tensor("op_1790")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2437 = const()[name = tensor("op_2437"), val = tensor([4, 4, 9, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_2420)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_2437, x = q_21)[name = tensor("q")]; - tensor var_2439 = const()[name = tensor("op_2439"), val = tensor([4, 4, 9, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_2427)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_2439, x = k_21)[name = tensor("k")]; - tensor var_2441 = const()[name = tensor("op_2441"), val = tensor([4, 4, 9, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_2434)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_2441, x = v_21)[name = tensor("v")]; + tensor var_1793 = const()[name = tensor("op_1793"), val = tensor([4, 4, 9, 64])]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1776)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1793, x = q_21)[name = tensor("q")]; + tensor var_1795 = const()[name = tensor("op_1795"), val = tensor([4, 4, 9, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1783)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1795, x = k_21)[name = tensor("k")]; + tensor var_1797 = const()[name = tensor("op_1797"), val = tensor([4, 4, 9, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1790)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1797, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -2147,50 +1688,50 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_2444 = const()[name = tensor("op_2444"), val = tensor([2, 0, 1, 3])]; - tensor var_2449 = const()[name = tensor("op_2449"), val = tensor([36, 256])]; - tensor var_2445 = transpose(perm = var_2444, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_2449, x = var_2445)[name = tensor("attn_output_11")]; + tensor var_1800 = const()[name = tensor("op_1800"), val = tensor([2, 0, 1, 3])]; + tensor var_1805 = const()[name = tensor("op_1805"), val = tensor([36, 256])]; + tensor var_1801 = transpose(perm = var_1800, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1805, x = var_1801)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_2453 = const()[name = tensor("op_2453"), val = tensor([9, 4, 256])]; - tensor attn_output = reshape(shape = var_2453, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1809 = const()[name = tensor("op_1809"), val = tensor([9, 4, 256])]; + tensor attn_output = reshape(shape = var_1809, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; - tensor input_313 = add(x = query_5, y = sa2_out)[name = tensor("input_313")]; - tensor input_315_axes_0 = const()[name = tensor("input_315_axes_0"), val = tensor([-1])]; - tensor input_315 = layer_norm(axes = input_315_axes_0, beta = decoder_norm21_1_bias, epsilon = var_2029, gamma = decoder_norm21_1_weight, x = input_313)[name = tensor("input_315")]; - tensor input_317 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_315)[name = tensor("linear_54")]; - tensor input_319 = relu(x = input_317)[name = tensor("input_319")]; - tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_319)[name = tensor("linear_55")]; - tensor input_321 = add(x = input_315, y = ffn_out)[name = tensor("input_321")]; + tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; + tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1384, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; + tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; + tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; + tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_2029, gamma = decoder_norm22_1_weight, x = input_321)[name = tensor("xt")]; - tensor var_2473 = const()[name = tensor("op_2473"), val = tensor([1, 4, 9, 256])]; - tensor input = reshape(shape = var_2473, x = xt)[name = tensor("input")]; - tensor var_2475 = const()[name = tensor("op_2475"), val = tensor([-1])]; - tensor var_2476 = reduce_l2_norm(axes = var_2475, keep_dims = var_2032, x = input)[name = tensor("op_2476")]; - tensor const_54 = const()[name = tensor("const_54"), val = tensor(0x1.fffffep+127)]; - tensor clip_37 = clip(alpha = var_2023, beta = const_54, x = var_2476)[name = tensor("clip_37")]; - tensor var_2478 = real_div(x = input, y = clip_37)[name = tensor("op_2478")]; - tensor concat_21 = const()[name = tensor("concat_21"), val = tensor([4, 1, 256])]; - tensor reshape_0 = reshape(shape = concat_21, x = emb)[name = tensor("reshape_0")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1384, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1829 = const()[name = tensor("op_1829"), val = tensor([1, 4, 9, 256])]; + tensor input = reshape(shape = var_1829, x = xt)[name = tensor("input")]; + tensor var_1831 = const()[name = tensor("op_1831"), val = tensor([-1])]; + tensor var_1832 = reduce_l2_norm(axes = var_1831, keep_dims = var_1387, x = input)[name = tensor("op_1832")]; + tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; + tensor clip_37 = clip(alpha = var_1379, beta = const_42, x = var_1832)[name = tensor("clip_37")]; + tensor var_1834 = real_div(x = input, y = clip_37)[name = tensor("op_1834")]; + tensor concat_22 = const()[name = tensor("concat_22"), val = tensor([4, 1, 256])]; + tensor reshape_0 = reshape(shape = concat_22, x = emb)[name = tensor("reshape_0")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_22 = const()[name = tensor("concat_22"), val = tensor([4, 256, 9])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_2478)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_22, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_23 = const()[name = tensor("concat_23"), val = tensor([4, 256, 9])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1834)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_23, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = reshape_0, y = reshape_1)[name = tensor("matmul_2")]; - tensor concat_25 = const()[name = tensor("concat_25"), val = tensor([1, 4, 9])]; - tensor reshape_2 = reshape(shape = concat_25, x = matmul_2)[name = tensor("reshape_2")]; + tensor concat_26 = const()[name = tensor("concat_26"), val = tensor([1, 4, 9])]; + tensor reshape_2 = reshape(shape = concat_26, x = matmul_2)[name = tensor("reshape_2")]; tensor output_begin_0 = const()[name = tensor("output_begin_0"), val = tensor([0, 0, 1])]; tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 4, 8])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = reshape_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_2482")]; - tensor var_2484_axis_0 = const()[name = tensor("op_2484_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_2484_axis_0, values = (var_2180, nkv))[name = tensor("op_2484")]; - tensor var_2486_axis_0 = const()[name = tensor("op_2486_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_2486_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_2486")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1838")]; + tensor var_1840_axis_0 = const()[name = tensor("op_1840_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1840_axis_0, values = (var_1536, nkv))[name = tensor("op_1840")]; + tensor var_1842_axis_0 = const()[name = tensor("op_1842_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1842_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1842")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/CALLHOME/ls_eend_ch_400ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/CALLHOME/ls_eend_ch_400ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index 79c878e4509cd4202322f9d93d708d1cbd90bec5..fe311308d7c3c006ed65220cdfe2949307b7019b 100644 --- a/optimized/CALLHOME/ls_eend_ch_400ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/CALLHOME/ls_eend_ch_400ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6160e50bd496653f6d5157337bcf04c0151de944864aa6ec466a20000c49b8a8 -size 311052 +oid sha256:59e17a3979d63a10da6e96e58a77c57329a6301355be13f029dd99c05bcf5482 +size 238957 diff --git a/optimized/CALLHOME/ls_eend_ch_400ms.mlpackage/Manifest.json b/optimized/CALLHOME/ls_eend_ch_400ms.mlpackage/Manifest.json index 2fe145faab20db18284325a23c4d73265a5feee8..dd6efdcc57920dd77b787174cc684af9a6514a57 100644 --- a/optimized/CALLHOME/ls_eend_ch_400ms.mlpackage/Manifest.json +++ b/optimized/CALLHOME/ls_eend_ch_400ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "6F71A32D-910A-4469-86B1-6350766893E1": { - "author": "com.apple.CoreML", - "description": "CoreML Model Specification", - "name": "model.mlmodel", - "path": "com.apple.CoreML/model.mlmodel" - }, - "AB34DCE1-F16B-4431-BD7A-EE499FF68470": { + "0C9CCAD6-063D-4F46-A212-7A360FDB7DE8": { "author": "com.apple.CoreML", "description": "CoreML Model Weights", "name": "weights", "path": "com.apple.CoreML/weights" + }, + "DA8F68BB-FC91-41C5-B90D-8CFB20D75827": { + "author": "com.apple.CoreML", + "description": "CoreML Model Specification", + "name": "model.mlmodel", + "path": "com.apple.CoreML/model.mlmodel" } }, - "rootModelIdentifier": "6F71A32D-910A-4469-86B1-6350766893E1" + "rootModelIdentifier": "DA8F68BB-FC91-41C5-B90D-8CFB20D75827" } diff --git a/optimized/CALLHOME/ls_eend_ch_500ms.mlmodelc/analytics/coremldata.bin b/optimized/CALLHOME/ls_eend_ch_500ms.mlmodelc/analytics/coremldata.bin index b9506ed574943a585aff41da1e912b0a7b0ade58..222b94a27daa8307ba766502db614666b15b0ecb 100644 --- a/optimized/CALLHOME/ls_eend_ch_500ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/CALLHOME/ls_eend_ch_500ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5a121adbc27573b4b63e5f3bf55273c531633caa84b8a875b546cc9ec0ebaaff +oid sha256:de89fe4bc4fc25d153a0fc3dbfeee67d789cecfc2c6722ec40c3c3a3a53ff76b size 243 diff --git a/optimized/CALLHOME/ls_eend_ch_500ms.mlmodelc/coremldata.bin b/optimized/CALLHOME/ls_eend_ch_500ms.mlmodelc/coremldata.bin index f6343f88fc2e75f287f86f69d847786bdd4406f6..49c147999e09e9d83b710547f3a13007f8a0678b 100644 --- a/optimized/CALLHOME/ls_eend_ch_500ms.mlmodelc/coremldata.bin +++ b/optimized/CALLHOME/ls_eend_ch_500ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e5431c384c91ebbe7e2a072b72cc2ba1cf5386e68185f036ef2baf942604fd94 -size 1259 +oid sha256:d1dc624f543d211720d14b7cfe3847efc38202fe9b80131c524fc811cd28b672 +size 1350 diff --git a/optimized/CALLHOME/ls_eend_ch_500ms.mlmodelc/metadata.json b/optimized/CALLHOME/ls_eend_ch_500ms.mlmodelc/metadata.json index 5a959ff8ee1e6c56eb48e5bb7735093e7a7135fb..e88235a314b252418f2b57efef1d473989486e5c 100644 --- a/optimized/CALLHOME/ls_eend_ch_500ms.mlmodelc/metadata.json +++ b/optimized/CALLHOME/ls_eend_ch_500ms.mlmodelc/metadata.json @@ -96,30 +96,30 @@ "Ios17.reshape" : 108, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 85, - "Split" : 20, + "Ios17.transpose" : 57, + "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 106, "Tile" : 21, - "Ios17.sliceByIndex" : 108, - "Ios16.sigmoid" : 21, + "Ios17.sliceByIndex" : 76, + "Ios16.sigmoid" : 5, "Ios16.reduceSum" : 1, "Ios17.squeeze" : 2, - "Ios17.layerNorm" : 49, - "Ios17.batchNorm" : 20, + "Ios17.layerNorm" : 33, + "Ios17.batchNorm" : 4, "Ios17.cast" : 40, "Ios17.sqrt" : 18, "Ios17.sub" : 40, - "Ios17.conv" : 61, + "Ios17.conv" : 13, "Ios17.realDiv" : 20, "Ios17.linear" : 56, "Ios17.concat" : 26, "Ios17.clip" : 46, "Ios17.greaterEqual" : 20, - "Ios16.silu" : 34, + "Ios16.silu" : 18, "Stack" : 6, "Ios16.cumsum" : 1, - "Ios17.mul" : 143 + "Ios17.mul" : 127 }, "computePrecision" : "Mixed (Float32, Int32)", "isUpdatable" : "0", @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"ch\", \"model_label\": \"CALLHOME\", \"variant\": \"pipeline\", \"T\": 5, \"max_speakers\": 7, \"max_nspks\": 9, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"ch\", \"model_label\": \"CALLHOME\", \"variant\": \"pipeline\", \"chunk_size\": 5, \"step_duration_ms\": 500, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 7, \"max_nspks\": 9, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/CALLHOME/ls_eend_ch_500ms.mlmodelc/model.mil b/optimized/CALLHOME/ls_eend_ch_500ms.mlmodelc/model.mil index e66a8c10a50e4213459dc34b310c1167fffdd235..26b07bd0330bb4f5071e8069eae80520d80655d6 100644 --- a/optimized/CALLHOME/ls_eend_ch_500ms.mlmodelc/model.mil +++ b/optimized/CALLHOME/ls_eend_ch_500ms.mlmodelc/model.mil @@ -217,6 +217,7 @@ program(1.0) tensor var_22 = const()[name = tensor("op_22"), val = tensor(0x0p+0)]; tensor var_25 = const()[name = tensor("op_25"), val = tensor(0x1.0c6f7ap-20)]; tensor var_28 = const()[name = tensor("op_28"), val = tensor(2)]; + tensor var_31 = const()[name = tensor("op_31"), val = tensor(0)]; tensor var_34 = const()[name = tensor("op_34"), val = tensor(1)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(0x1.4f8b58p-17)]; tensor var_37 = const()[name = tensor("op_37"), val = tensor(true)]; @@ -263,8 +264,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_181)[name = tensor("transpose_85")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_173)[name = tensor("transpose_86")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_181)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_173)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_197 = const()[name = tensor("op_197"), val = tensor([5, 1])]; tensor var_198 = reshape(shape = var_197, x = sqrt_s_t_1)[name = tensor("op_198")]; @@ -272,7 +273,7 @@ program(1.0) tensor var_200 = mul(x = qk_1, y = M_1)[name = tensor("op_200")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_187)[name = tensor("transpose_84")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_187)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_200, y = v_1)[name = tensor("inner_1")]; tensor var_202_transpose_x_0 = const()[name = tensor("op_202_transpose_x_0"), val = tensor(false)]; tensor var_202_transpose_y_0 = const()[name = tensor("op_202_transpose_y_0"), val = tensor(false)]; @@ -293,7 +294,7 @@ program(1.0) tensor var_215 = real_div(x = new_kv_unnorm_1, y = var_214)[name = tensor("op_215")]; tensor var_216_perm_0 = const()[name = tensor("op_216_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_216 = transpose(perm = var_216_perm_0, x = out_1)[name = tensor("transpose_83")]; + tensor var_216 = transpose(perm = var_216_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_25, x = var_216)[name = tensor("out_3")]; tensor var_220 = const()[name = tensor("op_220"), val = tensor([1, 5, 256])]; tensor out_5 = reshape(shape = var_220, x = out_3)[name = tensor("out_5")]; @@ -341,364 +342,210 @@ program(1.0) tensor var_253 = sub(x = var_18, y = use_roll_1)[name = tensor("op_253")]; tensor var_254 = mul(x = filled_window_1, y = var_253)[name = tensor("op_254")]; tensor var_255 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_255")]; - tensor input_21 = add(x = var_254, y = var_255)[name = tensor("input_21")]; + tensor window_3 = add(x = var_254, y = var_255)[name = tensor("window_3")]; tensor var_257 = const()[name = tensor("op_257"), val = tensor(0x1p+0)]; tensor var_258 = add(x = filled_1, y = var_257)[name = tensor("op_258")]; tensor clip_1 = clip(alpha = var_22, beta = var_17, x = var_258)[name = tensor("clip_1")]; + tensor var_261_begin_0 = const()[name = tensor("op_261_begin_0"), val = tensor([0, 1, 0])]; + tensor var_261_end_0 = const()[name = tensor("op_261_end_0"), val = tensor([1, 2, 256])]; + tensor var_261_end_mask_0 = const()[name = tensor("op_261_end_mask_0"), val = tensor([true, false, true])]; + tensor var_261 = slice_by_index(begin = var_261_begin_0, end = var_261_end_0, end_mask = var_261_end_mask_0, x = x_3)[name = tensor("op_261")]; + tensor var_264_begin_0 = const()[name = tensor("op_264_begin_0"), val = tensor([0, 1, 0])]; + tensor var_264_end_0 = const()[name = tensor("op_264_end_0"), val = tensor([1, 16, 256])]; + tensor var_264_end_mask_0 = const()[name = tensor("op_264_end_mask_0"), val = tensor([true, true, true])]; + tensor var_264 = slice_by_index(begin = var_264_begin_0, end = var_264_end_0, end_mask = var_264_end_mask_0, x = window_3)[name = tensor("op_264")]; + tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; + tensor rolled_3 = concat(axis = var_34, interleave = rolled_3_interleave_0, values = (var_264, var_261))[name = tensor("rolled_3")]; + tensor clip_2 = clip(alpha = var_22, beta = var_21, x = clip_1)[name = tensor("clip_2")]; + tensor var_269 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_269")]; + tensor var_270_dtype_0 = const()[name = tensor("op_270_dtype_0"), val = tensor("fp32")]; + tensor var_271 = const()[name = tensor("op_271"), val = tensor([1, 16, 1])]; + tensor var_270 = cast(dtype = var_270_dtype_0, x = var_269)[name = tensor("cast_189")]; + tensor fill_mask_3 = reshape(shape = var_271, x = var_270)[name = tensor("fill_mask_3")]; + tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_261)[name = tensor("x_expanded_3")]; + tensor var_275 = sub(x = var_18, y = fill_mask_3)[name = tensor("op_275")]; + tensor var_276 = mul(x = window_3, y = var_275)[name = tensor("op_276")]; + tensor var_277 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_277")]; + tensor filled_window_3 = add(x = var_276, y = var_277)[name = tensor("filled_window_3")]; + tensor var_279 = greater_equal(x = clip_1, y = var_17)[name = tensor("op_279")]; + tensor var_280_dtype_0 = const()[name = tensor("op_280_dtype_0"), val = tensor("fp32")]; + tensor var_281 = const()[name = tensor("op_281"), val = tensor([1, 1, 1])]; + tensor var_280 = cast(dtype = var_280_dtype_0, x = var_279)[name = tensor("cast_188")]; + tensor use_roll_3 = reshape(shape = var_281, x = var_280)[name = tensor("use_roll_3")]; + tensor var_283 = sub(x = var_18, y = use_roll_3)[name = tensor("op_283")]; + tensor var_284 = mul(x = filled_window_3, y = var_283)[name = tensor("op_284")]; + tensor var_285 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_285")]; + tensor window_5 = add(x = var_284, y = var_285)[name = tensor("window_5")]; + tensor var_287 = const()[name = tensor("op_287"), val = tensor(0x1p+0)]; + tensor var_288 = add(x = clip_1, y = var_287)[name = tensor("op_288")]; + tensor clip_3 = clip(alpha = var_22, beta = var_17, x = var_288)[name = tensor("clip_3")]; + tensor var_291_begin_0 = const()[name = tensor("op_291_begin_0"), val = tensor([0, 2, 0])]; + tensor var_291_end_0 = const()[name = tensor("op_291_end_0"), val = tensor([1, 3, 256])]; + tensor var_291_end_mask_0 = const()[name = tensor("op_291_end_mask_0"), val = tensor([true, false, true])]; + tensor var_291 = slice_by_index(begin = var_291_begin_0, end = var_291_end_0, end_mask = var_291_end_mask_0, x = x_3)[name = tensor("op_291")]; + tensor var_294_begin_0 = const()[name = tensor("op_294_begin_0"), val = tensor([0, 1, 0])]; + tensor var_294_end_0 = const()[name = tensor("op_294_end_0"), val = tensor([1, 16, 256])]; + tensor var_294_end_mask_0 = const()[name = tensor("op_294_end_mask_0"), val = tensor([true, true, true])]; + tensor var_294 = slice_by_index(begin = var_294_begin_0, end = var_294_end_0, end_mask = var_294_end_mask_0, x = window_5)[name = tensor("op_294")]; + tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; + tensor rolled_5 = concat(axis = var_34, interleave = rolled_5_interleave_0, values = (var_294, var_291))[name = tensor("rolled_5")]; + tensor clip_4 = clip(alpha = var_22, beta = var_21, x = clip_3)[name = tensor("clip_4")]; + tensor var_299 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_299")]; + tensor var_300_dtype_0 = const()[name = tensor("op_300_dtype_0"), val = tensor("fp32")]; + tensor var_301 = const()[name = tensor("op_301"), val = tensor([1, 16, 1])]; + tensor var_300 = cast(dtype = var_300_dtype_0, x = var_299)[name = tensor("cast_187")]; + tensor fill_mask_5 = reshape(shape = var_301, x = var_300)[name = tensor("fill_mask_5")]; + tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_291)[name = tensor("x_expanded_5")]; + tensor var_305 = sub(x = var_18, y = fill_mask_5)[name = tensor("op_305")]; + tensor var_306 = mul(x = window_5, y = var_305)[name = tensor("op_306")]; + tensor var_307 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_307")]; + tensor filled_window_5 = add(x = var_306, y = var_307)[name = tensor("filled_window_5")]; + tensor var_309 = greater_equal(x = clip_3, y = var_17)[name = tensor("op_309")]; + tensor var_310_dtype_0 = const()[name = tensor("op_310_dtype_0"), val = tensor("fp32")]; + tensor var_311 = const()[name = tensor("op_311"), val = tensor([1, 1, 1])]; + tensor var_310 = cast(dtype = var_310_dtype_0, x = var_309)[name = tensor("cast_186")]; + tensor use_roll_5 = reshape(shape = var_311, x = var_310)[name = tensor("use_roll_5")]; + tensor var_313 = sub(x = var_18, y = use_roll_5)[name = tensor("op_313")]; + tensor var_314 = mul(x = filled_window_5, y = var_313)[name = tensor("op_314")]; + tensor var_315 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_315")]; + tensor window_7 = add(x = var_314, y = var_315)[name = tensor("window_7")]; + tensor var_317 = const()[name = tensor("op_317"), val = tensor(0x1p+0)]; + tensor var_318 = add(x = clip_3, y = var_317)[name = tensor("op_318")]; + tensor clip_5 = clip(alpha = var_22, beta = var_17, x = var_318)[name = tensor("clip_5")]; + tensor var_321_begin_0 = const()[name = tensor("op_321_begin_0"), val = tensor([0, 3, 0])]; + tensor var_321_end_0 = const()[name = tensor("op_321_end_0"), val = tensor([1, 4, 256])]; + tensor var_321_end_mask_0 = const()[name = tensor("op_321_end_mask_0"), val = tensor([true, false, true])]; + tensor var_321 = slice_by_index(begin = var_321_begin_0, end = var_321_end_0, end_mask = var_321_end_mask_0, x = x_3)[name = tensor("op_321")]; + tensor var_324_begin_0 = const()[name = tensor("op_324_begin_0"), val = tensor([0, 1, 0])]; + tensor var_324_end_0 = const()[name = tensor("op_324_end_0"), val = tensor([1, 16, 256])]; + tensor var_324_end_mask_0 = const()[name = tensor("op_324_end_mask_0"), val = tensor([true, true, true])]; + tensor var_324 = slice_by_index(begin = var_324_begin_0, end = var_324_end_0, end_mask = var_324_end_mask_0, x = window_7)[name = tensor("op_324")]; + tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; + tensor rolled_7 = concat(axis = var_34, interleave = rolled_7_interleave_0, values = (var_324, var_321))[name = tensor("rolled_7")]; + tensor clip_6 = clip(alpha = var_22, beta = var_21, x = clip_5)[name = tensor("clip_6")]; + tensor var_329 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_329")]; + tensor var_330_dtype_0 = const()[name = tensor("op_330_dtype_0"), val = tensor("fp32")]; + tensor var_331 = const()[name = tensor("op_331"), val = tensor([1, 16, 1])]; + tensor var_330 = cast(dtype = var_330_dtype_0, x = var_329)[name = tensor("cast_185")]; + tensor fill_mask_7 = reshape(shape = var_331, x = var_330)[name = tensor("fill_mask_7")]; + tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_321)[name = tensor("x_expanded_7")]; + tensor var_335 = sub(x = var_18, y = fill_mask_7)[name = tensor("op_335")]; + tensor var_336 = mul(x = window_7, y = var_335)[name = tensor("op_336")]; + tensor var_337 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_337")]; + tensor filled_window_7 = add(x = var_336, y = var_337)[name = tensor("filled_window_7")]; + tensor var_339 = greater_equal(x = clip_5, y = var_17)[name = tensor("op_339")]; + tensor var_340_dtype_0 = const()[name = tensor("op_340_dtype_0"), val = tensor("fp32")]; + tensor var_341 = const()[name = tensor("op_341"), val = tensor([1, 1, 1])]; + tensor var_340 = cast(dtype = var_340_dtype_0, x = var_339)[name = tensor("cast_184")]; + tensor use_roll_7 = reshape(shape = var_341, x = var_340)[name = tensor("use_roll_7")]; + tensor var_343 = sub(x = var_18, y = use_roll_7)[name = tensor("op_343")]; + tensor var_344 = mul(x = filled_window_7, y = var_343)[name = tensor("op_344")]; + tensor var_345 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_345")]; + tensor window_9 = add(x = var_344, y = var_345)[name = tensor("window_9")]; + tensor var_347 = const()[name = tensor("op_347"), val = tensor(0x1p+0)]; + tensor var_348 = add(x = clip_5, y = var_347)[name = tensor("op_348")]; + tensor clip_7 = clip(alpha = var_22, beta = var_17, x = var_348)[name = tensor("clip_7")]; + tensor var_351_begin_0 = const()[name = tensor("op_351_begin_0"), val = tensor([0, 4, 0])]; + tensor var_351_end_0 = const()[name = tensor("op_351_end_0"), val = tensor([1, 1, 256])]; + tensor var_351_end_mask_0 = const()[name = tensor("op_351_end_mask_0"), val = tensor([true, true, true])]; + tensor var_351 = slice_by_index(begin = var_351_begin_0, end = var_351_end_0, end_mask = var_351_end_mask_0, x = x_3)[name = tensor("op_351")]; + tensor var_354_begin_0 = const()[name = tensor("op_354_begin_0"), val = tensor([0, 1, 0])]; + tensor var_354_end_0 = const()[name = tensor("op_354_end_0"), val = tensor([1, 16, 256])]; + tensor var_354_end_mask_0 = const()[name = tensor("op_354_end_mask_0"), val = tensor([true, true, true])]; + tensor var_354 = slice_by_index(begin = var_354_begin_0, end = var_354_end_0, end_mask = var_354_end_mask_0, x = window_9)[name = tensor("op_354")]; + tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; + tensor rolled_9 = concat(axis = var_34, interleave = rolled_9_interleave_0, values = (var_354, var_351))[name = tensor("rolled_9")]; + tensor clip_8 = clip(alpha = var_22, beta = var_21, x = clip_7)[name = tensor("clip_8")]; + tensor var_359 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_359")]; + tensor var_360_dtype_0 = const()[name = tensor("op_360_dtype_0"), val = tensor("fp32")]; + tensor var_361 = const()[name = tensor("op_361"), val = tensor([1, 16, 1])]; + tensor var_360 = cast(dtype = var_360_dtype_0, x = var_359)[name = tensor("cast_183")]; + tensor fill_mask_9 = reshape(shape = var_361, x = var_360)[name = tensor("fill_mask_9")]; + tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_351)[name = tensor("x_expanded_9")]; + tensor var_365 = sub(x = var_18, y = fill_mask_9)[name = tensor("op_365")]; + tensor var_366 = mul(x = window_9, y = var_365)[name = tensor("op_366")]; + tensor var_367 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_367")]; + tensor filled_window_9 = add(x = var_366, y = var_367)[name = tensor("filled_window_9")]; + tensor var_369 = greater_equal(x = clip_7, y = var_17)[name = tensor("op_369")]; + tensor var_370_dtype_0 = const()[name = tensor("op_370_dtype_0"), val = tensor("fp32")]; + tensor var_371 = const()[name = tensor("op_371"), val = tensor([1, 1, 1])]; + tensor var_370 = cast(dtype = var_370_dtype_0, x = var_369)[name = tensor("cast_182")]; + tensor use_roll_9 = reshape(shape = var_371, x = var_370)[name = tensor("use_roll_9")]; + tensor var_373 = sub(x = var_18, y = use_roll_9)[name = tensor("op_373")]; + tensor var_374 = mul(x = filled_window_9, y = var_373)[name = tensor("op_374")]; + tensor var_375 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_375")]; + tensor window_11 = add(x = var_374, y = var_375)[name = tensor("window_11")]; + tensor var_377 = const()[name = tensor("op_377"), val = tensor(0x1p+0)]; + tensor var_378 = add(x = clip_7, y = var_377)[name = tensor("op_378")]; + tensor clip_9 = clip(alpha = var_22, beta = var_17, x = var_378)[name = tensor("clip_9")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_31, interleave = input_21_interleave_0, values = (window_3, window_5, window_7, window_9, window_11))[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; - tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; + tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; tensor inputs_3_pad_type_0 = const()[name = tensor("inputs_3_pad_type_0"), val = tensor("valid")]; tensor inputs_3_strides_0 = const()[name = tensor("inputs_3_strides_0"), val = tensor([1])]; tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_82")]; - tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_279_split_sizes_0 = const()[name = tensor("op_279_split_sizes_0"), val = tensor([256, 256])]; - tensor var_279_axis_0 = const()[name = tensor("op_279_axis_0"), val = tensor(1)]; - tensor var_279_0, tensor var_279_1 = split(axis = var_279_axis_0, split_sizes = var_279_split_sizes_0, x = inputs_3)[name = tensor("op_279")]; - tensor var_281 = sigmoid(x = var_279_1)[name = tensor("op_281")]; - tensor inputs_5 = mul(x = var_279_0, y = var_281)[name = tensor("inputs_5")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; + tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; + tensor var_401_split_sizes_0 = const()[name = tensor("op_401_split_sizes_0"), val = tensor([256, 256])]; + tensor var_401_axis_0 = const()[name = tensor("op_401_axis_0"), val = tensor(1)]; + tensor var_401_0, tensor var_401_1 = split(axis = var_401_axis_0, split_sizes = var_401_split_sizes_0, x = inputs_3)[name = tensor("op_401")]; + tensor var_403 = sigmoid(x = var_401_1)[name = tensor("op_403")]; + tensor inputs_5 = mul(x = var_401_0, y = var_403)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; tensor outputs_aug_1_strides_0 = const()[name = tensor("outputs_aug_1_strides_0"), val = tensor([1])]; tensor outputs_aug_1_dilations_0 = const()[name = tensor("outputs_aug_1_dilations_0"), val = tensor([1])]; - tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; + tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; tensor input_25_begin_0 = const()[name = tensor("input_25_begin_0"), val = tensor([0, 0, 0])]; - tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([1, 256, 16])]; + tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([5, 256, 16])]; tensor input_25_end_mask_0 = const()[name = tensor("input_25_end_mask_0"), val = tensor([true, true, false])]; - tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; - tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; - tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; + tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; + tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; + tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; tensor input_29_pad_type_0 = const()[name = tensor("input_29_pad_type_0"), val = tensor("valid")]; tensor input_29_strides_0 = const()[name = tensor("input_29_strides_0"), val = tensor([1])]; tensor input_29_pad_0 = const()[name = tensor("input_29_pad_0"), val = tensor([0, 0])]; tensor input_29_dilations_0 = const()[name = tensor("input_29_dilations_0"), val = tensor([1])]; tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; - tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; + tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_312_begin_0 = const()[name = tensor("op_312_begin_0"), val = tensor([0, -1, 0])]; - tensor var_312_end_0 = const()[name = tensor("op_312_end_0"), val = tensor([1, 16, 256])]; - tensor var_312_end_mask_0 = const()[name = tensor("op_312_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_81")]; - tensor var_312 = slice_by_index(begin = var_312_begin_0, end = var_312_end_0, end_mask = var_312_end_mask_0, x = conv_out_1)[name = tensor("op_312")]; - tensor var_315_begin_0 = const()[name = tensor("op_315_begin_0"), val = tensor([0, 1, 0])]; - tensor var_315_end_0 = const()[name = tensor("op_315_end_0"), val = tensor([1, 2, 256])]; - tensor var_315_end_mask_0 = const()[name = tensor("op_315_end_mask_0"), val = tensor([true, false, true])]; - tensor var_315 = slice_by_index(begin = var_315_begin_0, end = var_315_end_0, end_mask = var_315_end_mask_0, x = x_3)[name = tensor("op_315")]; - tensor var_318_begin_0 = const()[name = tensor("op_318_begin_0"), val = tensor([0, 1, 0])]; - tensor var_318_end_0 = const()[name = tensor("op_318_end_0"), val = tensor([1, 16, 256])]; - tensor var_318_end_mask_0 = const()[name = tensor("op_318_end_mask_0"), val = tensor([true, true, true])]; - tensor var_318 = slice_by_index(begin = var_318_begin_0, end = var_318_end_0, end_mask = var_318_end_mask_0, x = input_21)[name = tensor("op_318")]; - tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_34, interleave = rolled_3_interleave_0, values = (var_318, var_315))[name = tensor("rolled_3")]; - tensor clip_2 = clip(alpha = var_22, beta = var_21, x = clip_1)[name = tensor("clip_2")]; - tensor var_323 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_323")]; - tensor var_324_dtype_0 = const()[name = tensor("op_324_dtype_0"), val = tensor("fp32")]; - tensor var_325 = const()[name = tensor("op_325"), val = tensor([1, 16, 1])]; - tensor var_324 = cast(dtype = var_324_dtype_0, x = var_323)[name = tensor("cast_189")]; - tensor fill_mask_3 = reshape(shape = var_325, x = var_324)[name = tensor("fill_mask_3")]; - tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_315)[name = tensor("x_expanded_3")]; - tensor var_329 = sub(x = var_18, y = fill_mask_3)[name = tensor("op_329")]; - tensor var_330 = mul(x = input_21, y = var_329)[name = tensor("op_330")]; - tensor var_331 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_331")]; - tensor filled_window_3 = add(x = var_330, y = var_331)[name = tensor("filled_window_3")]; - tensor var_333 = greater_equal(x = clip_1, y = var_17)[name = tensor("op_333")]; - tensor var_334_dtype_0 = const()[name = tensor("op_334_dtype_0"), val = tensor("fp32")]; - tensor var_335 = const()[name = tensor("op_335"), val = tensor([1, 1, 1])]; - tensor var_334 = cast(dtype = var_334_dtype_0, x = var_333)[name = tensor("cast_188")]; - tensor use_roll_3 = reshape(shape = var_335, x = var_334)[name = tensor("use_roll_3")]; - tensor var_337 = sub(x = var_18, y = use_roll_3)[name = tensor("op_337")]; - tensor var_338 = mul(x = filled_window_3, y = var_337)[name = tensor("op_338")]; - tensor var_339 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_339")]; - tensor input_31 = add(x = var_338, y = var_339)[name = tensor("input_31")]; - tensor var_341 = const()[name = tensor("op_341"), val = tensor(0x1p+0)]; - tensor var_342 = add(x = clip_1, y = var_341)[name = tensor("op_342")]; - tensor clip_3 = clip(alpha = var_22, beta = var_17, x = var_342)[name = tensor("clip_3")]; + tensor var_434_begin_0 = const()[name = tensor("op_434_begin_0"), val = tensor([0, -1, 0])]; + tensor var_434_end_0 = const()[name = tensor("op_434_end_0"), val = tensor([5, 16, 256])]; + tensor var_434_end_mask_0 = const()[name = tensor("op_434_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_434 = slice_by_index(begin = var_434_begin_0, end = var_434_end_0, end_mask = var_434_end_mask_0, x = conv_out_1)[name = tensor("op_434")]; + tensor var_436_perm_0 = const()[name = tensor("op_436_perm_0"), val = tensor([1, 0, 2])]; + tensor var_436 = transpose(perm = var_436_perm_0, x = var_434)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_436)[name = tensor("input_31")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; + tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; + tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; + tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; + tensor var_459 = const()[name = tensor("op_459"), val = tensor(0x1p-1)]; + tensor var_460 = mul(x = input_39, y = var_459)[name = tensor("op_460")]; + tensor input_41 = add(x = var_460, y = input_31)[name = tensor("input_41")]; + tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; + tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_36, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; + tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; + tensor input_45 = layer_norm(axes = input_45_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_43)[name = tensor("input_45")]; + tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; + tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; + tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; + tensor var_489 = const()[name = tensor("op_489"), val = tensor(0x1p-1)]; + tensor var_490 = mul(x = input_51, y = var_489)[name = tensor("op_490")]; + tensor input_53 = add(x = var_490, y = input_43)[name = tensor("input_53")]; tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; - tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_31)[name = tensor("x_7")]; - tensor input_33_perm_0 = const()[name = tensor("input_33_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_9_pad_type_0 = const()[name = tensor("inputs_9_pad_type_0"), val = tensor("valid")]; - tensor inputs_9_strides_0 = const()[name = tensor("inputs_9_strides_0"), val = tensor([1])]; - tensor inputs_9_pad_0 = const()[name = tensor("inputs_9_pad_0"), val = tensor([0, 0])]; - tensor inputs_9_dilations_0 = const()[name = tensor("inputs_9_dilations_0"), val = tensor([1])]; - tensor inputs_9_groups_0 = const()[name = tensor("inputs_9_groups_0"), val = tensor(1)]; - tensor input_33 = transpose(perm = input_33_perm_0, x = x_7)[name = tensor("transpose_80")]; - tensor inputs_9 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_9_dilations_0, groups = inputs_9_groups_0, pad = inputs_9_pad_0, pad_type = inputs_9_pad_type_0, strides = inputs_9_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_33)[name = tensor("inputs_9")]; - tensor var_363_split_sizes_0 = const()[name = tensor("op_363_split_sizes_0"), val = tensor([256, 256])]; - tensor var_363_axis_0 = const()[name = tensor("op_363_axis_0"), val = tensor(1)]; - tensor var_363_0, tensor var_363_1 = split(axis = var_363_axis_0, split_sizes = var_363_split_sizes_0, x = inputs_9)[name = tensor("op_363")]; - tensor var_365 = sigmoid(x = var_363_1)[name = tensor("op_365")]; - tensor inputs_11 = mul(x = var_363_0, y = var_365)[name = tensor("inputs_11")]; - tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; - tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; - tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; - tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_11)[name = tensor("outputs_aug_3")]; - tensor input_35_begin_0 = const()[name = tensor("input_35_begin_0"), val = tensor([0, 0, 0])]; - tensor input_35_end_0 = const()[name = tensor("input_35_end_0"), val = tensor([1, 256, 16])]; - tensor input_35_end_mask_0 = const()[name = tensor("input_35_end_mask_0"), val = tensor([true, true, false])]; - tensor input_35 = slice_by_index(begin = input_35_begin_0, end = input_35_end_0, end_mask = input_35_end_mask_0, x = outputs_aug_3)[name = tensor("input_35")]; - tensor inputs_13 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_35)[name = tensor("inputs_13")]; - tensor input_37 = silu(x = inputs_13)[name = tensor("input_37")]; - tensor input_39_pad_type_0 = const()[name = tensor("input_39_pad_type_0"), val = tensor("valid")]; - tensor input_39_strides_0 = const()[name = tensor("input_39_strides_0"), val = tensor([1])]; - tensor input_39_pad_0 = const()[name = tensor("input_39_pad_0"), val = tensor([0, 0])]; - tensor input_39_dilations_0 = const()[name = tensor("input_39_dilations_0"), val = tensor([1])]; - tensor input_39_groups_0 = const()[name = tensor("input_39_groups_0"), val = tensor(1)]; - tensor input_39 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_37)[name = tensor("input_39")]; - tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_396_begin_0 = const()[name = tensor("op_396_begin_0"), val = tensor([0, -1, 0])]; - tensor var_396_end_0 = const()[name = tensor("op_396_end_0"), val = tensor([1, 16, 256])]; - tensor var_396_end_mask_0 = const()[name = tensor("op_396_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_39)[name = tensor("transpose_79")]; - tensor var_396 = slice_by_index(begin = var_396_begin_0, end = var_396_end_0, end_mask = var_396_end_mask_0, x = conv_out_3)[name = tensor("op_396")]; - tensor var_399_begin_0 = const()[name = tensor("op_399_begin_0"), val = tensor([0, 2, 0])]; - tensor var_399_end_0 = const()[name = tensor("op_399_end_0"), val = tensor([1, 3, 256])]; - tensor var_399_end_mask_0 = const()[name = tensor("op_399_end_mask_0"), val = tensor([true, false, true])]; - tensor var_399 = slice_by_index(begin = var_399_begin_0, end = var_399_end_0, end_mask = var_399_end_mask_0, x = x_3)[name = tensor("op_399")]; - tensor var_402_begin_0 = const()[name = tensor("op_402_begin_0"), val = tensor([0, 1, 0])]; - tensor var_402_end_0 = const()[name = tensor("op_402_end_0"), val = tensor([1, 16, 256])]; - tensor var_402_end_mask_0 = const()[name = tensor("op_402_end_mask_0"), val = tensor([true, true, true])]; - tensor var_402 = slice_by_index(begin = var_402_begin_0, end = var_402_end_0, end_mask = var_402_end_mask_0, x = input_31)[name = tensor("op_402")]; - tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_34, interleave = rolled_5_interleave_0, values = (var_402, var_399))[name = tensor("rolled_5")]; - tensor clip_4 = clip(alpha = var_22, beta = var_21, x = clip_3)[name = tensor("clip_4")]; - tensor var_407 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_407")]; - tensor var_408_dtype_0 = const()[name = tensor("op_408_dtype_0"), val = tensor("fp32")]; - tensor var_409 = const()[name = tensor("op_409"), val = tensor([1, 16, 1])]; - tensor var_408 = cast(dtype = var_408_dtype_0, x = var_407)[name = tensor("cast_187")]; - tensor fill_mask_5 = reshape(shape = var_409, x = var_408)[name = tensor("fill_mask_5")]; - tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_399)[name = tensor("x_expanded_5")]; - tensor var_413 = sub(x = var_18, y = fill_mask_5)[name = tensor("op_413")]; - tensor var_414 = mul(x = input_31, y = var_413)[name = tensor("op_414")]; - tensor var_415 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_415")]; - tensor filled_window_5 = add(x = var_414, y = var_415)[name = tensor("filled_window_5")]; - tensor var_417 = greater_equal(x = clip_3, y = var_17)[name = tensor("op_417")]; - tensor var_418_dtype_0 = const()[name = tensor("op_418_dtype_0"), val = tensor("fp32")]; - tensor var_419 = const()[name = tensor("op_419"), val = tensor([1, 1, 1])]; - tensor var_418 = cast(dtype = var_418_dtype_0, x = var_417)[name = tensor("cast_186")]; - tensor use_roll_5 = reshape(shape = var_419, x = var_418)[name = tensor("use_roll_5")]; - tensor var_421 = sub(x = var_18, y = use_roll_5)[name = tensor("op_421")]; - tensor var_422 = mul(x = filled_window_5, y = var_421)[name = tensor("op_422")]; - tensor var_423 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_423")]; - tensor input_41 = add(x = var_422, y = var_423)[name = tensor("input_41")]; - tensor var_425 = const()[name = tensor("op_425"), val = tensor(0x1p+0)]; - tensor var_426 = add(x = clip_3, y = var_425)[name = tensor("op_426")]; - tensor clip_5 = clip(alpha = var_22, beta = var_17, x = var_426)[name = tensor("clip_5")]; - tensor x_9_axes_0 = const()[name = tensor("x_9_axes_0"), val = tensor([-1])]; - tensor x_9 = layer_norm(axes = x_9_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_41)[name = tensor("x_9")]; - tensor input_43_perm_0 = const()[name = tensor("input_43_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_15_pad_type_0 = const()[name = tensor("inputs_15_pad_type_0"), val = tensor("valid")]; - tensor inputs_15_strides_0 = const()[name = tensor("inputs_15_strides_0"), val = tensor([1])]; - tensor inputs_15_pad_0 = const()[name = tensor("inputs_15_pad_0"), val = tensor([0, 0])]; - tensor inputs_15_dilations_0 = const()[name = tensor("inputs_15_dilations_0"), val = tensor([1])]; - tensor inputs_15_groups_0 = const()[name = tensor("inputs_15_groups_0"), val = tensor(1)]; - tensor input_43 = transpose(perm = input_43_perm_0, x = x_9)[name = tensor("transpose_78")]; - tensor inputs_15 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_15_dilations_0, groups = inputs_15_groups_0, pad = inputs_15_pad_0, pad_type = inputs_15_pad_type_0, strides = inputs_15_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_43)[name = tensor("inputs_15")]; - tensor var_447_split_sizes_0 = const()[name = tensor("op_447_split_sizes_0"), val = tensor([256, 256])]; - tensor var_447_axis_0 = const()[name = tensor("op_447_axis_0"), val = tensor(1)]; - tensor var_447_0, tensor var_447_1 = split(axis = var_447_axis_0, split_sizes = var_447_split_sizes_0, x = inputs_15)[name = tensor("op_447")]; - tensor var_449 = sigmoid(x = var_447_1)[name = tensor("op_449")]; - tensor inputs_17 = mul(x = var_447_0, y = var_449)[name = tensor("inputs_17")]; - tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; - tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; - tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; - tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_17)[name = tensor("outputs_aug_5")]; - tensor input_45_begin_0 = const()[name = tensor("input_45_begin_0"), val = tensor([0, 0, 0])]; - tensor input_45_end_0 = const()[name = tensor("input_45_end_0"), val = tensor([1, 256, 16])]; - tensor input_45_end_mask_0 = const()[name = tensor("input_45_end_mask_0"), val = tensor([true, true, false])]; - tensor input_45 = slice_by_index(begin = input_45_begin_0, end = input_45_end_0, end_mask = input_45_end_mask_0, x = outputs_aug_5)[name = tensor("input_45")]; - tensor inputs_19 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_45)[name = tensor("inputs_19")]; - tensor input_47 = silu(x = inputs_19)[name = tensor("input_47")]; - tensor input_49_pad_type_0 = const()[name = tensor("input_49_pad_type_0"), val = tensor("valid")]; - tensor input_49_strides_0 = const()[name = tensor("input_49_strides_0"), val = tensor([1])]; - tensor input_49_pad_0 = const()[name = tensor("input_49_pad_0"), val = tensor([0, 0])]; - tensor input_49_dilations_0 = const()[name = tensor("input_49_dilations_0"), val = tensor([1])]; - tensor input_49_groups_0 = const()[name = tensor("input_49_groups_0"), val = tensor(1)]; - tensor input_49 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_49_dilations_0, groups = input_49_groups_0, pad = input_49_pad_0, pad_type = input_49_pad_type_0, strides = input_49_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_47)[name = tensor("input_49")]; - tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_480_begin_0 = const()[name = tensor("op_480_begin_0"), val = tensor([0, -1, 0])]; - tensor var_480_end_0 = const()[name = tensor("op_480_end_0"), val = tensor([1, 16, 256])]; - tensor var_480_end_mask_0 = const()[name = tensor("op_480_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_49)[name = tensor("transpose_77")]; - tensor var_480 = slice_by_index(begin = var_480_begin_0, end = var_480_end_0, end_mask = var_480_end_mask_0, x = conv_out_5)[name = tensor("op_480")]; - tensor var_483_begin_0 = const()[name = tensor("op_483_begin_0"), val = tensor([0, 3, 0])]; - tensor var_483_end_0 = const()[name = tensor("op_483_end_0"), val = tensor([1, 4, 256])]; - tensor var_483_end_mask_0 = const()[name = tensor("op_483_end_mask_0"), val = tensor([true, false, true])]; - tensor var_483 = slice_by_index(begin = var_483_begin_0, end = var_483_end_0, end_mask = var_483_end_mask_0, x = x_3)[name = tensor("op_483")]; - tensor var_486_begin_0 = const()[name = tensor("op_486_begin_0"), val = tensor([0, 1, 0])]; - tensor var_486_end_0 = const()[name = tensor("op_486_end_0"), val = tensor([1, 16, 256])]; - tensor var_486_end_mask_0 = const()[name = tensor("op_486_end_mask_0"), val = tensor([true, true, true])]; - tensor var_486 = slice_by_index(begin = var_486_begin_0, end = var_486_end_0, end_mask = var_486_end_mask_0, x = input_41)[name = tensor("op_486")]; - tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; - tensor rolled_7 = concat(axis = var_34, interleave = rolled_7_interleave_0, values = (var_486, var_483))[name = tensor("rolled_7")]; - tensor clip_6 = clip(alpha = var_22, beta = var_21, x = clip_5)[name = tensor("clip_6")]; - tensor var_491 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_491")]; - tensor var_492_dtype_0 = const()[name = tensor("op_492_dtype_0"), val = tensor("fp32")]; - tensor var_493 = const()[name = tensor("op_493"), val = tensor([1, 16, 1])]; - tensor var_492 = cast(dtype = var_492_dtype_0, x = var_491)[name = tensor("cast_185")]; - tensor fill_mask_7 = reshape(shape = var_493, x = var_492)[name = tensor("fill_mask_7")]; - tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_483)[name = tensor("x_expanded_7")]; - tensor var_497 = sub(x = var_18, y = fill_mask_7)[name = tensor("op_497")]; - tensor var_498 = mul(x = input_41, y = var_497)[name = tensor("op_498")]; - tensor var_499 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_499")]; - tensor filled_window_7 = add(x = var_498, y = var_499)[name = tensor("filled_window_7")]; - tensor var_501 = greater_equal(x = clip_5, y = var_17)[name = tensor("op_501")]; - tensor var_502_dtype_0 = const()[name = tensor("op_502_dtype_0"), val = tensor("fp32")]; - tensor var_503 = const()[name = tensor("op_503"), val = tensor([1, 1, 1])]; - tensor var_502 = cast(dtype = var_502_dtype_0, x = var_501)[name = tensor("cast_184")]; - tensor use_roll_7 = reshape(shape = var_503, x = var_502)[name = tensor("use_roll_7")]; - tensor var_505 = sub(x = var_18, y = use_roll_7)[name = tensor("op_505")]; - tensor var_506 = mul(x = filled_window_7, y = var_505)[name = tensor("op_506")]; - tensor var_507 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_507")]; - tensor input_51 = add(x = var_506, y = var_507)[name = tensor("input_51")]; - tensor var_509 = const()[name = tensor("op_509"), val = tensor(0x1p+0)]; - tensor var_510 = add(x = clip_5, y = var_509)[name = tensor("op_510")]; - tensor clip_7 = clip(alpha = var_22, beta = var_17, x = var_510)[name = tensor("clip_7")]; - tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; - tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_51)[name = tensor("x_11")]; - tensor input_53_perm_0 = const()[name = tensor("input_53_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_21_pad_type_0 = const()[name = tensor("inputs_21_pad_type_0"), val = tensor("valid")]; - tensor inputs_21_strides_0 = const()[name = tensor("inputs_21_strides_0"), val = tensor([1])]; - tensor inputs_21_pad_0 = const()[name = tensor("inputs_21_pad_0"), val = tensor([0, 0])]; - tensor inputs_21_dilations_0 = const()[name = tensor("inputs_21_dilations_0"), val = tensor([1])]; - tensor inputs_21_groups_0 = const()[name = tensor("inputs_21_groups_0"), val = tensor(1)]; - tensor input_53 = transpose(perm = input_53_perm_0, x = x_11)[name = tensor("transpose_76")]; - tensor inputs_21 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_21_dilations_0, groups = inputs_21_groups_0, pad = inputs_21_pad_0, pad_type = inputs_21_pad_type_0, strides = inputs_21_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_53)[name = tensor("inputs_21")]; - tensor var_531_split_sizes_0 = const()[name = tensor("op_531_split_sizes_0"), val = tensor([256, 256])]; - tensor var_531_axis_0 = const()[name = tensor("op_531_axis_0"), val = tensor(1)]; - tensor var_531_0, tensor var_531_1 = split(axis = var_531_axis_0, split_sizes = var_531_split_sizes_0, x = inputs_21)[name = tensor("op_531")]; - tensor var_533 = sigmoid(x = var_531_1)[name = tensor("op_533")]; - tensor inputs_23 = mul(x = var_531_0, y = var_533)[name = tensor("inputs_23")]; - tensor outputs_aug_7_pad_type_0 = const()[name = tensor("outputs_aug_7_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_7_pad_0 = const()[name = tensor("outputs_aug_7_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_7_groups_0 = const()[name = tensor("outputs_aug_7_groups_0"), val = tensor(256)]; - tensor outputs_aug_7_strides_0 = const()[name = tensor("outputs_aug_7_strides_0"), val = tensor([1])]; - tensor outputs_aug_7_dilations_0 = const()[name = tensor("outputs_aug_7_dilations_0"), val = tensor([1])]; - tensor outputs_aug_7 = conv(dilations = outputs_aug_7_dilations_0, groups = outputs_aug_7_groups_0, pad = outputs_aug_7_pad_0, pad_type = outputs_aug_7_pad_type_0, strides = outputs_aug_7_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_23)[name = tensor("outputs_aug_7")]; - tensor input_55_begin_0 = const()[name = tensor("input_55_begin_0"), val = tensor([0, 0, 0])]; - tensor input_55_end_0 = const()[name = tensor("input_55_end_0"), val = tensor([1, 256, 16])]; - tensor input_55_end_mask_0 = const()[name = tensor("input_55_end_mask_0"), val = tensor([true, true, false])]; - tensor input_55 = slice_by_index(begin = input_55_begin_0, end = input_55_end_0, end_mask = input_55_end_mask_0, x = outputs_aug_7)[name = tensor("input_55")]; - tensor inputs_25 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_55)[name = tensor("inputs_25")]; - tensor input_57 = silu(x = inputs_25)[name = tensor("input_57")]; - tensor input_59_pad_type_0 = const()[name = tensor("input_59_pad_type_0"), val = tensor("valid")]; - tensor input_59_strides_0 = const()[name = tensor("input_59_strides_0"), val = tensor([1])]; - tensor input_59_pad_0 = const()[name = tensor("input_59_pad_0"), val = tensor([0, 0])]; - tensor input_59_dilations_0 = const()[name = tensor("input_59_dilations_0"), val = tensor([1])]; - tensor input_59_groups_0 = const()[name = tensor("input_59_groups_0"), val = tensor(1)]; - tensor input_59 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_59_dilations_0, groups = input_59_groups_0, pad = input_59_pad_0, pad_type = input_59_pad_type_0, strides = input_59_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_57)[name = tensor("input_59")]; - tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_564_begin_0 = const()[name = tensor("op_564_begin_0"), val = tensor([0, -1, 0])]; - tensor var_564_end_0 = const()[name = tensor("op_564_end_0"), val = tensor([1, 16, 256])]; - tensor var_564_end_mask_0 = const()[name = tensor("op_564_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_59)[name = tensor("transpose_75")]; - tensor var_564 = slice_by_index(begin = var_564_begin_0, end = var_564_end_0, end_mask = var_564_end_mask_0, x = conv_out_7)[name = tensor("op_564")]; - tensor var_567_begin_0 = const()[name = tensor("op_567_begin_0"), val = tensor([0, 4, 0])]; - tensor var_567_end_0 = const()[name = tensor("op_567_end_0"), val = tensor([1, 1, 256])]; - tensor var_567_end_mask_0 = const()[name = tensor("op_567_end_mask_0"), val = tensor([true, true, true])]; - tensor var_567 = slice_by_index(begin = var_567_begin_0, end = var_567_end_0, end_mask = var_567_end_mask_0, x = x_3)[name = tensor("op_567")]; - tensor var_570_begin_0 = const()[name = tensor("op_570_begin_0"), val = tensor([0, 1, 0])]; - tensor var_570_end_0 = const()[name = tensor("op_570_end_0"), val = tensor([1, 16, 256])]; - tensor var_570_end_mask_0 = const()[name = tensor("op_570_end_mask_0"), val = tensor([true, true, true])]; - tensor var_570 = slice_by_index(begin = var_570_begin_0, end = var_570_end_0, end_mask = var_570_end_mask_0, x = input_51)[name = tensor("op_570")]; - tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; - tensor rolled_9 = concat(axis = var_34, interleave = rolled_9_interleave_0, values = (var_570, var_567))[name = tensor("rolled_9")]; - tensor clip_8 = clip(alpha = var_22, beta = var_21, x = clip_7)[name = tensor("clip_8")]; - tensor var_575 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_575")]; - tensor var_576_dtype_0 = const()[name = tensor("op_576_dtype_0"), val = tensor("fp32")]; - tensor var_577 = const()[name = tensor("op_577"), val = tensor([1, 16, 1])]; - tensor var_576 = cast(dtype = var_576_dtype_0, x = var_575)[name = tensor("cast_183")]; - tensor fill_mask_9 = reshape(shape = var_577, x = var_576)[name = tensor("fill_mask_9")]; - tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_567)[name = tensor("x_expanded_9")]; - tensor var_581 = sub(x = var_18, y = fill_mask_9)[name = tensor("op_581")]; - tensor var_582 = mul(x = input_51, y = var_581)[name = tensor("op_582")]; - tensor var_583 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_583")]; - tensor filled_window_9 = add(x = var_582, y = var_583)[name = tensor("filled_window_9")]; - tensor var_585 = greater_equal(x = clip_7, y = var_17)[name = tensor("op_585")]; - tensor var_586_dtype_0 = const()[name = tensor("op_586_dtype_0"), val = tensor("fp32")]; - tensor var_587 = const()[name = tensor("op_587"), val = tensor([1, 1, 1])]; - tensor var_586 = cast(dtype = var_586_dtype_0, x = var_585)[name = tensor("cast_182")]; - tensor use_roll_9 = reshape(shape = var_587, x = var_586)[name = tensor("use_roll_9")]; - tensor var_589 = sub(x = var_18, y = use_roll_9)[name = tensor("op_589")]; - tensor var_590 = mul(x = filled_window_9, y = var_589)[name = tensor("op_590")]; - tensor var_591 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_591")]; - tensor input_61 = add(x = var_590, y = var_591)[name = tensor("input_61")]; - tensor var_593 = const()[name = tensor("op_593"), val = tensor(0x1p+0)]; - tensor var_594 = add(x = clip_7, y = var_593)[name = tensor("op_594")]; - tensor clip_9 = clip(alpha = var_22, beta = var_17, x = var_594)[name = tensor("clip_9")]; - tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; - tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_61)[name = tensor("x_13")]; - tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_27_pad_type_0 = const()[name = tensor("inputs_27_pad_type_0"), val = tensor("valid")]; - tensor inputs_27_strides_0 = const()[name = tensor("inputs_27_strides_0"), val = tensor([1])]; - tensor inputs_27_pad_0 = const()[name = tensor("inputs_27_pad_0"), val = tensor([0, 0])]; - tensor inputs_27_dilations_0 = const()[name = tensor("inputs_27_dilations_0"), val = tensor([1])]; - tensor inputs_27_groups_0 = const()[name = tensor("inputs_27_groups_0"), val = tensor(1)]; - tensor input_63 = transpose(perm = input_63_perm_0, x = x_13)[name = tensor("transpose_74")]; - tensor inputs_27 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_27_dilations_0, groups = inputs_27_groups_0, pad = inputs_27_pad_0, pad_type = inputs_27_pad_type_0, strides = inputs_27_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_27")]; - tensor var_615_split_sizes_0 = const()[name = tensor("op_615_split_sizes_0"), val = tensor([256, 256])]; - tensor var_615_axis_0 = const()[name = tensor("op_615_axis_0"), val = tensor(1)]; - tensor var_615_0, tensor var_615_1 = split(axis = var_615_axis_0, split_sizes = var_615_split_sizes_0, x = inputs_27)[name = tensor("op_615")]; - tensor var_617 = sigmoid(x = var_615_1)[name = tensor("op_617")]; - tensor inputs_29 = mul(x = var_615_0, y = var_617)[name = tensor("inputs_29")]; - tensor outputs_aug_9_pad_type_0 = const()[name = tensor("outputs_aug_9_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_9_pad_0 = const()[name = tensor("outputs_aug_9_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_9_groups_0 = const()[name = tensor("outputs_aug_9_groups_0"), val = tensor(256)]; - tensor outputs_aug_9_strides_0 = const()[name = tensor("outputs_aug_9_strides_0"), val = tensor([1])]; - tensor outputs_aug_9_dilations_0 = const()[name = tensor("outputs_aug_9_dilations_0"), val = tensor([1])]; - tensor outputs_aug_9 = conv(dilations = outputs_aug_9_dilations_0, groups = outputs_aug_9_groups_0, pad = outputs_aug_9_pad_0, pad_type = outputs_aug_9_pad_type_0, strides = outputs_aug_9_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_29)[name = tensor("outputs_aug_9")]; - tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; - tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([1, 256, 16])]; - tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; - tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_9)[name = tensor("input_65")]; - tensor inputs_31 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_65)[name = tensor("inputs_31")]; - tensor input_67 = silu(x = inputs_31)[name = tensor("input_67")]; - tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; - tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; - tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; - tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; - tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; - tensor input_69 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; - tensor conv_out_9_perm_0 = const()[name = tensor("conv_out_9_perm_0"), val = tensor([0, 2, 1])]; - tensor var_648_begin_0 = const()[name = tensor("op_648_begin_0"), val = tensor([0, -1, 0])]; - tensor var_648_end_0 = const()[name = tensor("op_648_end_0"), val = tensor([1, 16, 256])]; - tensor var_648_end_mask_0 = const()[name = tensor("op_648_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_9 = transpose(perm = conv_out_9_perm_0, x = input_69)[name = tensor("transpose_73")]; - tensor var_648 = slice_by_index(begin = var_648_begin_0, end = var_648_end_0, end_mask = var_648_end_mask_0, x = conv_out_9)[name = tensor("op_648")]; - tensor var_651_interleave_0 = const()[name = tensor("op_651_interleave_0"), val = tensor(false)]; - tensor var_651 = concat(axis = var_34, interleave = var_651_interleave_0, values = (var_312, var_396, var_480, var_564, var_648))[name = tensor("op_651")]; - tensor input_71 = add(x = x_3, y = var_651)[name = tensor("input_71")]; - tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; - tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; - tensor inputs_33 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_8")]; - tensor input_75 = silu(x = inputs_33)[name = tensor("input_75")]; - tensor input_79 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_9")]; - tensor var_674 = const()[name = tensor("op_674"), val = tensor(0x1p-1)]; - tensor var_675 = mul(x = input_79, y = var_674)[name = tensor("op_675")]; - tensor input_81 = add(x = var_675, y = input_71)[name = tensor("input_81")]; - tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; - tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_36, gamma = encoder_layer_norm_0_weight, x = input_81)[name = tensor("input_83")]; - tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; - tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; - tensor inputs_35 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_10")]; - tensor input_87 = silu(x = inputs_35)[name = tensor("input_87")]; - tensor input_91 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_11")]; - tensor var_704 = const()[name = tensor("op_704"), val = tensor(0x1p-1)]; - tensor var_705 = mul(x = input_91, y = var_704)[name = tensor("op_705")]; - tensor input_93 = add(x = var_705, y = input_83)[name = tensor("input_93")]; - tensor x_15_axes_0 = const()[name = tensor("x_15_axes_0"), val = tensor([-1])]; - tensor x_15 = layer_norm(axes = x_15_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_36, gamma = encoder_ret_lns_1_weight, x = input_93)[name = tensor("x_15")]; + tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_36, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_3_end_0 = const()[name = tensor("prev_kv_3_end_0"), val = tensor([2, 1, 4, 64, 64])]; tensor prev_kv_3_end_mask_0 = const()[name = tensor("prev_kv_3_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -709,462 +556,308 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_719 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_15)[name = tensor("linear_12")]; - tensor var_720 = const()[name = tensor("op_720"), val = tensor([1, 5, 4, 64])]; - tensor var_721 = reshape(shape = var_720, x = var_719)[name = tensor("op_721")]; + tensor var_504 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_505 = const()[name = tensor("op_505"), val = tensor([1, 5, 4, 64])]; + tensor var_506 = reshape(shape = var_505, x = var_504)[name = tensor("op_506")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_725 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_15)[name = tensor("linear_13")]; - tensor var_726 = const()[name = tensor("op_726"), val = tensor(0x1p-3)]; - tensor var_727 = mul(x = var_725, y = var_726)[name = tensor("op_727")]; - tensor var_728 = const()[name = tensor("op_728"), val = tensor([1, 5, 4, 64])]; - tensor var_729 = reshape(shape = var_728, x = var_727)[name = tensor("op_729")]; + tensor var_510 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_511 = const()[name = tensor("op_511"), val = tensor(0x1p-3)]; + tensor var_512 = mul(x = var_510, y = var_511)[name = tensor("op_512")]; + tensor var_513 = const()[name = tensor("op_513"), val = tensor([1, 5, 4, 64])]; + tensor var_514 = reshape(shape = var_513, x = var_512)[name = tensor("op_514")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_733 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_15)[name = tensor("linear_14")]; - tensor var_734 = const()[name = tensor("op_734"), val = tensor([1, 5, 4, 64])]; - tensor var_735 = reshape(shape = var_734, x = var_733)[name = tensor("op_735")]; + tensor var_518 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_519 = const()[name = tensor("op_519"), val = tensor([1, 5, 4, 64])]; + tensor var_520 = reshape(shape = var_519, x = var_518)[name = tensor("op_520")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_97 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_15)[name = tensor("linear_15")]; + tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; tensor s_t_3 = add(x = prev_scale_3, y = encoder__t_index)[name = tensor("s_t_3")]; tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_729)[name = tensor("transpose_71")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_721)[name = tensor("transpose_72")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_514)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_506)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_745 = const()[name = tensor("op_745"), val = tensor([5, 1])]; - tensor var_746 = reshape(shape = var_745, x = sqrt_s_t_3)[name = tensor("op_746")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_746)[name = tensor("M_3")]; - tensor var_748 = mul(x = qk_3, y = M_3)[name = tensor("op_748")]; + tensor var_530 = const()[name = tensor("op_530"), val = tensor([5, 1])]; + tensor var_531 = reshape(shape = var_530, x = sqrt_s_t_3)[name = tensor("op_531")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_531)[name = tensor("M_3")]; + tensor var_533 = mul(x = qk_3, y = M_3)[name = tensor("op_533")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_735)[name = tensor("transpose_70")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_748, y = v_3)[name = tensor("inner_3")]; - tensor var_750_transpose_x_0 = const()[name = tensor("op_750_transpose_x_0"), val = tensor(false)]; - tensor var_750_transpose_y_0 = const()[name = tensor("op_750_transpose_y_0"), val = tensor(false)]; - tensor var_750 = matmul(transpose_x = var_750_transpose_x_0, transpose_y = var_750_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_750")]; - tensor var_751 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_751")]; - tensor var_752 = const()[name = tensor("op_752"), val = tensor([1, 1, 5, 1])]; - tensor var_753 = reshape(shape = var_752, x = var_751)[name = tensor("op_753")]; - tensor cross_3 = mul(x = var_750, y = var_753)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_520)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_533, y = v_3)[name = tensor("inner_3")]; + tensor var_535_transpose_x_0 = const()[name = tensor("op_535_transpose_x_0"), val = tensor(false)]; + tensor var_535_transpose_y_0 = const()[name = tensor("op_535_transpose_y_0"), val = tensor(false)]; + tensor var_535 = matmul(transpose_x = var_535_transpose_x_0, transpose_y = var_535_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_535")]; + tensor var_536 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_536")]; + tensor var_537 = const()[name = tensor("op_537"), val = tensor([1, 1, 5, 1])]; + tensor var_538 = reshape(shape = var_537, x = var_536)[name = tensor("op_538")]; + tensor cross_3 = mul(x = var_535, y = var_538)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_756 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_756")]; - tensor var_758_transpose_x_1 = const()[name = tensor("op_758_transpose_x_1"), val = tensor(true)]; - tensor var_758_transpose_y_1 = const()[name = tensor("op_758_transpose_y_1"), val = tensor(false)]; - tensor var_758 = matmul(transpose_x = var_758_transpose_x_1, transpose_y = var_758_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_758")]; - tensor new_kv_unnorm_3 = add(x = var_756, y = var_758)[name = tensor("new_kv_unnorm_3")]; - tensor var_760 = const()[name = tensor("op_760"), val = tensor(0x1.4p+2)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_760)[name = tensor("new_scale_3")]; - tensor var_762 = sqrt(x = new_scale_3)[name = tensor("op_762")]; - tensor var_763 = real_div(x = new_kv_unnorm_3, y = var_762)[name = tensor("op_763")]; - tensor var_764_perm_0 = const()[name = tensor("op_764_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_541 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_541")]; + tensor var_543_transpose_x_1 = const()[name = tensor("op_543_transpose_x_1"), val = tensor(true)]; + tensor var_543_transpose_y_1 = const()[name = tensor("op_543_transpose_y_1"), val = tensor(false)]; + tensor var_543 = matmul(transpose_x = var_543_transpose_x_1, transpose_y = var_543_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_543")]; + tensor new_kv_unnorm_3 = add(x = var_541, y = var_543)[name = tensor("new_kv_unnorm_3")]; + tensor var_545 = const()[name = tensor("op_545"), val = tensor(0x1.4p+2)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_545)[name = tensor("new_scale_3")]; + tensor var_547 = sqrt(x = new_scale_3)[name = tensor("op_547")]; + tensor var_548 = real_div(x = new_kv_unnorm_3, y = var_547)[name = tensor("op_548")]; + tensor var_549_perm_0 = const()[name = tensor("op_549_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_764 = transpose(perm = var_764_perm_0, x = out_7)[name = tensor("transpose_69")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_25, x = var_764)[name = tensor("out_9")]; - tensor var_768 = const()[name = tensor("op_768"), val = tensor([1, 5, 256])]; - tensor out_11 = reshape(shape = var_768, x = out_9)[name = tensor("out_11")]; - tensor var_770 = silu(x = input_97)[name = tensor("op_770")]; - tensor input_99 = mul(x = var_770, y = out_11)[name = tensor("input_99")]; - tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_99)[name = tensor("linear_16")]; - tensor x_17 = add(x = input_93, y = ret_out_3)[name = tensor("x_17")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor var_549 = transpose(perm = var_549_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_25, x = var_549)[name = tensor("out_9")]; + tensor var_553 = const()[name = tensor("op_553"), val = tensor([1, 5, 256])]; + tensor out_11 = reshape(shape = var_553, x = out_9)[name = tensor("out_11")]; + tensor var_555 = silu(x = input_57)[name = tensor("op_555")]; + tensor input_59 = mul(x = var_555, y = out_11)[name = tensor("input_59")]; + tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; + tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; + tensor window_13_begin_0 = const()[name = tensor("window_13_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_13_end_0 = const()[name = tensor("window_13_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_13_end_mask_0 = const()[name = tensor("window_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_13_squeeze_mask_0 = const()[name = tensor("window_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_13 = slice_by_index(begin = window_13_begin_0, end = window_13_end_0, end_mask = window_13_end_mask_0, squeeze_mask = window_13_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_13")]; tensor filled_11_begin_0 = const()[name = tensor("filled_11_begin_0"), val = tensor([1, 0])]; tensor filled_11_end_0 = const()[name = tensor("filled_11_end_0"), val = tensor([2, 1])]; tensor filled_11_end_mask_0 = const()[name = tensor("filled_11_end_mask_0"), val = tensor([false, true])]; tensor filled_11_squeeze_mask_0 = const()[name = tensor("filled_11_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_11 = slice_by_index(begin = filled_11_begin_0, end = filled_11_end_0, end_mask = filled_11_end_mask_0, squeeze_mask = filled_11_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_11")]; - tensor var_779_begin_0 = const()[name = tensor("op_779_begin_0"), val = tensor([0, 0, 0])]; - tensor var_779_end_0 = const()[name = tensor("op_779_end_0"), val = tensor([1, 1, 256])]; - tensor var_779_end_mask_0 = const()[name = tensor("op_779_end_mask_0"), val = tensor([true, false, true])]; - tensor var_779 = slice_by_index(begin = var_779_begin_0, end = var_779_end_0, end_mask = var_779_end_mask_0, x = x_17)[name = tensor("op_779")]; - tensor var_782_begin_0 = const()[name = tensor("op_782_begin_0"), val = tensor([0, 1, 0])]; - tensor var_782_end_0 = const()[name = tensor("op_782_end_0"), val = tensor([1, 16, 256])]; - tensor var_782_end_mask_0 = const()[name = tensor("op_782_end_mask_0"), val = tensor([true, true, true])]; - tensor var_782 = slice_by_index(begin = var_782_begin_0, end = var_782_end_0, end_mask = var_782_end_mask_0, x = window_3)[name = tensor("op_782")]; + tensor var_564_begin_0 = const()[name = tensor("op_564_begin_0"), val = tensor([0, 0, 0])]; + tensor var_564_end_0 = const()[name = tensor("op_564_end_0"), val = tensor([1, 1, 256])]; + tensor var_564_end_mask_0 = const()[name = tensor("op_564_end_mask_0"), val = tensor([true, false, true])]; + tensor var_564 = slice_by_index(begin = var_564_begin_0, end = var_564_end_0, end_mask = var_564_end_mask_0, x = x_9)[name = tensor("op_564")]; + tensor var_567_begin_0 = const()[name = tensor("op_567_begin_0"), val = tensor([0, 1, 0])]; + tensor var_567_end_0 = const()[name = tensor("op_567_end_0"), val = tensor([1, 16, 256])]; + tensor var_567_end_mask_0 = const()[name = tensor("op_567_end_mask_0"), val = tensor([true, true, true])]; + tensor var_567 = slice_by_index(begin = var_567_begin_0, end = var_567_end_0, end_mask = var_567_end_mask_0, x = window_13)[name = tensor("op_567")]; tensor rolled_11_interleave_0 = const()[name = tensor("rolled_11_interleave_0"), val = tensor(false)]; - tensor rolled_11 = concat(axis = var_34, interleave = rolled_11_interleave_0, values = (var_782, var_779))[name = tensor("rolled_11")]; + tensor rolled_11 = concat(axis = var_34, interleave = rolled_11_interleave_0, values = (var_567, var_564))[name = tensor("rolled_11")]; tensor clip_10 = clip(alpha = var_22, beta = var_21, x = filled_11)[name = tensor("clip_10")]; - tensor var_787 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_787")]; - tensor var_788_dtype_0 = const()[name = tensor("op_788_dtype_0"), val = tensor("fp32")]; - tensor var_789 = const()[name = tensor("op_789"), val = tensor([1, 16, 1])]; - tensor var_788 = cast(dtype = var_788_dtype_0, x = var_787)[name = tensor("cast_181")]; - tensor fill_mask_11 = reshape(shape = var_789, x = var_788)[name = tensor("fill_mask_11")]; + tensor var_572 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_572")]; + tensor var_573_dtype_0 = const()[name = tensor("op_573_dtype_0"), val = tensor("fp32")]; + tensor var_574 = const()[name = tensor("op_574"), val = tensor([1, 16, 1])]; + tensor var_573 = cast(dtype = var_573_dtype_0, x = var_572)[name = tensor("cast_181")]; + tensor fill_mask_11 = reshape(shape = var_574, x = var_573)[name = tensor("fill_mask_11")]; tensor x_expanded_11_reps_0 = const()[name = tensor("x_expanded_11_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_779)[name = tensor("x_expanded_11")]; - tensor var_793 = sub(x = var_18, y = fill_mask_11)[name = tensor("op_793")]; - tensor var_794 = mul(x = window_3, y = var_793)[name = tensor("op_794")]; - tensor var_795 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_795")]; - tensor filled_window_11 = add(x = var_794, y = var_795)[name = tensor("filled_window_11")]; - tensor var_797 = greater_equal(x = filled_11, y = var_17)[name = tensor("op_797")]; - tensor var_798_dtype_0 = const()[name = tensor("op_798_dtype_0"), val = tensor("fp32")]; - tensor var_799 = const()[name = tensor("op_799"), val = tensor([1, 1, 1])]; - tensor var_798 = cast(dtype = var_798_dtype_0, x = var_797)[name = tensor("cast_180")]; - tensor use_roll_11 = reshape(shape = var_799, x = var_798)[name = tensor("use_roll_11")]; - tensor var_801 = sub(x = var_18, y = use_roll_11)[name = tensor("op_801")]; - tensor var_802 = mul(x = filled_window_11, y = var_801)[name = tensor("op_802")]; - tensor var_803 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_803")]; - tensor input_101 = add(x = var_802, y = var_803)[name = tensor("input_101")]; - tensor var_805 = const()[name = tensor("op_805"), val = tensor(0x1p+0)]; - tensor var_806 = add(x = filled_11, y = var_805)[name = tensor("op_806")]; - tensor clip_11 = clip(alpha = var_22, beta = var_17, x = var_806)[name = tensor("clip_11")]; - tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; - tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_101)[name = tensor("x_19")]; - tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_37_pad_type_0 = const()[name = tensor("inputs_37_pad_type_0"), val = tensor("valid")]; - tensor inputs_37_strides_0 = const()[name = tensor("inputs_37_strides_0"), val = tensor([1])]; - tensor inputs_37_pad_0 = const()[name = tensor("inputs_37_pad_0"), val = tensor([0, 0])]; - tensor inputs_37_dilations_0 = const()[name = tensor("inputs_37_dilations_0"), val = tensor([1])]; - tensor inputs_37_groups_0 = const()[name = tensor("inputs_37_groups_0"), val = tensor(1)]; - tensor input_103 = transpose(perm = input_103_perm_0, x = x_19)[name = tensor("transpose_68")]; - tensor inputs_37 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_37_dilations_0, groups = inputs_37_groups_0, pad = inputs_37_pad_0, pad_type = inputs_37_pad_type_0, strides = inputs_37_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_37")]; - tensor var_827_split_sizes_0 = const()[name = tensor("op_827_split_sizes_0"), val = tensor([256, 256])]; - tensor var_827_axis_0 = const()[name = tensor("op_827_axis_0"), val = tensor(1)]; - tensor var_827_0, tensor var_827_1 = split(axis = var_827_axis_0, split_sizes = var_827_split_sizes_0, x = inputs_37)[name = tensor("op_827")]; - tensor var_829 = sigmoid(x = var_827_1)[name = tensor("op_829")]; - tensor inputs_39 = mul(x = var_827_0, y = var_829)[name = tensor("inputs_39")]; - tensor outputs_aug_11_pad_type_0 = const()[name = tensor("outputs_aug_11_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_11_pad_0 = const()[name = tensor("outputs_aug_11_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_11_groups_0 = const()[name = tensor("outputs_aug_11_groups_0"), val = tensor(256)]; - tensor outputs_aug_11_strides_0 = const()[name = tensor("outputs_aug_11_strides_0"), val = tensor([1])]; - tensor outputs_aug_11_dilations_0 = const()[name = tensor("outputs_aug_11_dilations_0"), val = tensor([1])]; - tensor outputs_aug_11 = conv(dilations = outputs_aug_11_dilations_0, groups = outputs_aug_11_groups_0, pad = outputs_aug_11_pad_0, pad_type = outputs_aug_11_pad_type_0, strides = outputs_aug_11_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_39)[name = tensor("outputs_aug_11")]; - tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; - tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([1, 256, 16])]; - tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; - tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_11)[name = tensor("input_105")]; - tensor inputs_41 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_105)[name = tensor("inputs_41")]; - tensor input_107 = silu(x = inputs_41)[name = tensor("input_107")]; - tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; - tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; - tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; - tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; - tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; - tensor input_109 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; - tensor conv_out_11_perm_0 = const()[name = tensor("conv_out_11_perm_0"), val = tensor([0, 2, 1])]; - tensor var_860_begin_0 = const()[name = tensor("op_860_begin_0"), val = tensor([0, -1, 0])]; - tensor var_860_end_0 = const()[name = tensor("op_860_end_0"), val = tensor([1, 16, 256])]; - tensor var_860_end_mask_0 = const()[name = tensor("op_860_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_11 = transpose(perm = conv_out_11_perm_0, x = input_109)[name = tensor("transpose_67")]; - tensor var_860 = slice_by_index(begin = var_860_begin_0, end = var_860_end_0, end_mask = var_860_end_mask_0, x = conv_out_11)[name = tensor("op_860")]; - tensor var_863_begin_0 = const()[name = tensor("op_863_begin_0"), val = tensor([0, 1, 0])]; - tensor var_863_end_0 = const()[name = tensor("op_863_end_0"), val = tensor([1, 2, 256])]; - tensor var_863_end_mask_0 = const()[name = tensor("op_863_end_mask_0"), val = tensor([true, false, true])]; - tensor var_863 = slice_by_index(begin = var_863_begin_0, end = var_863_end_0, end_mask = var_863_end_mask_0, x = x_17)[name = tensor("op_863")]; - tensor var_866_begin_0 = const()[name = tensor("op_866_begin_0"), val = tensor([0, 1, 0])]; - tensor var_866_end_0 = const()[name = tensor("op_866_end_0"), val = tensor([1, 16, 256])]; - tensor var_866_end_mask_0 = const()[name = tensor("op_866_end_mask_0"), val = tensor([true, true, true])]; - tensor var_866 = slice_by_index(begin = var_866_begin_0, end = var_866_end_0, end_mask = var_866_end_mask_0, x = input_101)[name = tensor("op_866")]; + tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_564)[name = tensor("x_expanded_11")]; + tensor var_578 = sub(x = var_18, y = fill_mask_11)[name = tensor("op_578")]; + tensor var_579 = mul(x = window_13, y = var_578)[name = tensor("op_579")]; + tensor var_580 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_580")]; + tensor filled_window_11 = add(x = var_579, y = var_580)[name = tensor("filled_window_11")]; + tensor var_582 = greater_equal(x = filled_11, y = var_17)[name = tensor("op_582")]; + tensor var_583_dtype_0 = const()[name = tensor("op_583_dtype_0"), val = tensor("fp32")]; + tensor var_584 = const()[name = tensor("op_584"), val = tensor([1, 1, 1])]; + tensor var_583 = cast(dtype = var_583_dtype_0, x = var_582)[name = tensor("cast_180")]; + tensor use_roll_11 = reshape(shape = var_584, x = var_583)[name = tensor("use_roll_11")]; + tensor var_586 = sub(x = var_18, y = use_roll_11)[name = tensor("op_586")]; + tensor var_587 = mul(x = filled_window_11, y = var_586)[name = tensor("op_587")]; + tensor var_588 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_588")]; + tensor window_15 = add(x = var_587, y = var_588)[name = tensor("window_15")]; + tensor var_590 = const()[name = tensor("op_590"), val = tensor(0x1p+0)]; + tensor var_591 = add(x = filled_11, y = var_590)[name = tensor("op_591")]; + tensor clip_11 = clip(alpha = var_22, beta = var_17, x = var_591)[name = tensor("clip_11")]; + tensor var_594_begin_0 = const()[name = tensor("op_594_begin_0"), val = tensor([0, 1, 0])]; + tensor var_594_end_0 = const()[name = tensor("op_594_end_0"), val = tensor([1, 2, 256])]; + tensor var_594_end_mask_0 = const()[name = tensor("op_594_end_mask_0"), val = tensor([true, false, true])]; + tensor var_594 = slice_by_index(begin = var_594_begin_0, end = var_594_end_0, end_mask = var_594_end_mask_0, x = x_9)[name = tensor("op_594")]; + tensor var_597_begin_0 = const()[name = tensor("op_597_begin_0"), val = tensor([0, 1, 0])]; + tensor var_597_end_0 = const()[name = tensor("op_597_end_0"), val = tensor([1, 16, 256])]; + tensor var_597_end_mask_0 = const()[name = tensor("op_597_end_mask_0"), val = tensor([true, true, true])]; + tensor var_597 = slice_by_index(begin = var_597_begin_0, end = var_597_end_0, end_mask = var_597_end_mask_0, x = window_15)[name = tensor("op_597")]; tensor rolled_13_interleave_0 = const()[name = tensor("rolled_13_interleave_0"), val = tensor(false)]; - tensor rolled_13 = concat(axis = var_34, interleave = rolled_13_interleave_0, values = (var_866, var_863))[name = tensor("rolled_13")]; + tensor rolled_13 = concat(axis = var_34, interleave = rolled_13_interleave_0, values = (var_597, var_594))[name = tensor("rolled_13")]; tensor clip_12 = clip(alpha = var_22, beta = var_21, x = clip_11)[name = tensor("clip_12")]; - tensor var_871 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_871")]; - tensor var_872_dtype_0 = const()[name = tensor("op_872_dtype_0"), val = tensor("fp32")]; - tensor var_873 = const()[name = tensor("op_873"), val = tensor([1, 16, 1])]; - tensor var_872 = cast(dtype = var_872_dtype_0, x = var_871)[name = tensor("cast_179")]; - tensor fill_mask_13 = reshape(shape = var_873, x = var_872)[name = tensor("fill_mask_13")]; + tensor var_602 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_602")]; + tensor var_603_dtype_0 = const()[name = tensor("op_603_dtype_0"), val = tensor("fp32")]; + tensor var_604 = const()[name = tensor("op_604"), val = tensor([1, 16, 1])]; + tensor var_603 = cast(dtype = var_603_dtype_0, x = var_602)[name = tensor("cast_179")]; + tensor fill_mask_13 = reshape(shape = var_604, x = var_603)[name = tensor("fill_mask_13")]; tensor x_expanded_13_reps_0 = const()[name = tensor("x_expanded_13_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_863)[name = tensor("x_expanded_13")]; - tensor var_877 = sub(x = var_18, y = fill_mask_13)[name = tensor("op_877")]; - tensor var_878 = mul(x = input_101, y = var_877)[name = tensor("op_878")]; - tensor var_879 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_879")]; - tensor filled_window_13 = add(x = var_878, y = var_879)[name = tensor("filled_window_13")]; - tensor var_881 = greater_equal(x = clip_11, y = var_17)[name = tensor("op_881")]; - tensor var_882_dtype_0 = const()[name = tensor("op_882_dtype_0"), val = tensor("fp32")]; - tensor var_883 = const()[name = tensor("op_883"), val = tensor([1, 1, 1])]; - tensor var_882 = cast(dtype = var_882_dtype_0, x = var_881)[name = tensor("cast_178")]; - tensor use_roll_13 = reshape(shape = var_883, x = var_882)[name = tensor("use_roll_13")]; - tensor var_885 = sub(x = var_18, y = use_roll_13)[name = tensor("op_885")]; - tensor var_886 = mul(x = filled_window_13, y = var_885)[name = tensor("op_886")]; - tensor var_887 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_887")]; - tensor input_111 = add(x = var_886, y = var_887)[name = tensor("input_111")]; - tensor var_889 = const()[name = tensor("op_889"), val = tensor(0x1p+0)]; - tensor var_890 = add(x = clip_11, y = var_889)[name = tensor("op_890")]; - tensor clip_13 = clip(alpha = var_22, beta = var_17, x = var_890)[name = tensor("clip_13")]; - tensor x_21_axes_0 = const()[name = tensor("x_21_axes_0"), val = tensor([-1])]; - tensor x_21 = layer_norm(axes = x_21_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_111)[name = tensor("x_21")]; - tensor input_113_perm_0 = const()[name = tensor("input_113_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_43_pad_type_0 = const()[name = tensor("inputs_43_pad_type_0"), val = tensor("valid")]; - tensor inputs_43_strides_0 = const()[name = tensor("inputs_43_strides_0"), val = tensor([1])]; - tensor inputs_43_pad_0 = const()[name = tensor("inputs_43_pad_0"), val = tensor([0, 0])]; - tensor inputs_43_dilations_0 = const()[name = tensor("inputs_43_dilations_0"), val = tensor([1])]; - tensor inputs_43_groups_0 = const()[name = tensor("inputs_43_groups_0"), val = tensor(1)]; - tensor input_113 = transpose(perm = input_113_perm_0, x = x_21)[name = tensor("transpose_66")]; - tensor inputs_43 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_43_dilations_0, groups = inputs_43_groups_0, pad = inputs_43_pad_0, pad_type = inputs_43_pad_type_0, strides = inputs_43_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_113)[name = tensor("inputs_43")]; - tensor var_911_split_sizes_0 = const()[name = tensor("op_911_split_sizes_0"), val = tensor([256, 256])]; - tensor var_911_axis_0 = const()[name = tensor("op_911_axis_0"), val = tensor(1)]; - tensor var_911_0, tensor var_911_1 = split(axis = var_911_axis_0, split_sizes = var_911_split_sizes_0, x = inputs_43)[name = tensor("op_911")]; - tensor var_913 = sigmoid(x = var_911_1)[name = tensor("op_913")]; - tensor inputs_45 = mul(x = var_911_0, y = var_913)[name = tensor("inputs_45")]; - tensor outputs_aug_13_pad_type_0 = const()[name = tensor("outputs_aug_13_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_13_pad_0 = const()[name = tensor("outputs_aug_13_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_13_groups_0 = const()[name = tensor("outputs_aug_13_groups_0"), val = tensor(256)]; - tensor outputs_aug_13_strides_0 = const()[name = tensor("outputs_aug_13_strides_0"), val = tensor([1])]; - tensor outputs_aug_13_dilations_0 = const()[name = tensor("outputs_aug_13_dilations_0"), val = tensor([1])]; - tensor outputs_aug_13 = conv(dilations = outputs_aug_13_dilations_0, groups = outputs_aug_13_groups_0, pad = outputs_aug_13_pad_0, pad_type = outputs_aug_13_pad_type_0, strides = outputs_aug_13_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_45)[name = tensor("outputs_aug_13")]; - tensor input_115_begin_0 = const()[name = tensor("input_115_begin_0"), val = tensor([0, 0, 0])]; - tensor input_115_end_0 = const()[name = tensor("input_115_end_0"), val = tensor([1, 256, 16])]; - tensor input_115_end_mask_0 = const()[name = tensor("input_115_end_mask_0"), val = tensor([true, true, false])]; - tensor input_115 = slice_by_index(begin = input_115_begin_0, end = input_115_end_0, end_mask = input_115_end_mask_0, x = outputs_aug_13)[name = tensor("input_115")]; - tensor inputs_47 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_115)[name = tensor("inputs_47")]; - tensor input_117 = silu(x = inputs_47)[name = tensor("input_117")]; - tensor input_119_pad_type_0 = const()[name = tensor("input_119_pad_type_0"), val = tensor("valid")]; - tensor input_119_strides_0 = const()[name = tensor("input_119_strides_0"), val = tensor([1])]; - tensor input_119_pad_0 = const()[name = tensor("input_119_pad_0"), val = tensor([0, 0])]; - tensor input_119_dilations_0 = const()[name = tensor("input_119_dilations_0"), val = tensor([1])]; - tensor input_119_groups_0 = const()[name = tensor("input_119_groups_0"), val = tensor(1)]; - tensor input_119 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_119_dilations_0, groups = input_119_groups_0, pad = input_119_pad_0, pad_type = input_119_pad_type_0, strides = input_119_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_117)[name = tensor("input_119")]; - tensor conv_out_13_perm_0 = const()[name = tensor("conv_out_13_perm_0"), val = tensor([0, 2, 1])]; - tensor var_944_begin_0 = const()[name = tensor("op_944_begin_0"), val = tensor([0, -1, 0])]; - tensor var_944_end_0 = const()[name = tensor("op_944_end_0"), val = tensor([1, 16, 256])]; - tensor var_944_end_mask_0 = const()[name = tensor("op_944_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_13 = transpose(perm = conv_out_13_perm_0, x = input_119)[name = tensor("transpose_65")]; - tensor var_944 = slice_by_index(begin = var_944_begin_0, end = var_944_end_0, end_mask = var_944_end_mask_0, x = conv_out_13)[name = tensor("op_944")]; - tensor var_947_begin_0 = const()[name = tensor("op_947_begin_0"), val = tensor([0, 2, 0])]; - tensor var_947_end_0 = const()[name = tensor("op_947_end_0"), val = tensor([1, 3, 256])]; - tensor var_947_end_mask_0 = const()[name = tensor("op_947_end_mask_0"), val = tensor([true, false, true])]; - tensor var_947 = slice_by_index(begin = var_947_begin_0, end = var_947_end_0, end_mask = var_947_end_mask_0, x = x_17)[name = tensor("op_947")]; - tensor var_950_begin_0 = const()[name = tensor("op_950_begin_0"), val = tensor([0, 1, 0])]; - tensor var_950_end_0 = const()[name = tensor("op_950_end_0"), val = tensor([1, 16, 256])]; - tensor var_950_end_mask_0 = const()[name = tensor("op_950_end_mask_0"), val = tensor([true, true, true])]; - tensor var_950 = slice_by_index(begin = var_950_begin_0, end = var_950_end_0, end_mask = var_950_end_mask_0, x = input_111)[name = tensor("op_950")]; + tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_594)[name = tensor("x_expanded_13")]; + tensor var_608 = sub(x = var_18, y = fill_mask_13)[name = tensor("op_608")]; + tensor var_609 = mul(x = window_15, y = var_608)[name = tensor("op_609")]; + tensor var_610 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_610")]; + tensor filled_window_13 = add(x = var_609, y = var_610)[name = tensor("filled_window_13")]; + tensor var_612 = greater_equal(x = clip_11, y = var_17)[name = tensor("op_612")]; + tensor var_613_dtype_0 = const()[name = tensor("op_613_dtype_0"), val = tensor("fp32")]; + tensor var_614 = const()[name = tensor("op_614"), val = tensor([1, 1, 1])]; + tensor var_613 = cast(dtype = var_613_dtype_0, x = var_612)[name = tensor("cast_178")]; + tensor use_roll_13 = reshape(shape = var_614, x = var_613)[name = tensor("use_roll_13")]; + tensor var_616 = sub(x = var_18, y = use_roll_13)[name = tensor("op_616")]; + tensor var_617 = mul(x = filled_window_13, y = var_616)[name = tensor("op_617")]; + tensor var_618 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_618")]; + tensor window_17 = add(x = var_617, y = var_618)[name = tensor("window_17")]; + tensor var_620 = const()[name = tensor("op_620"), val = tensor(0x1p+0)]; + tensor var_621 = add(x = clip_11, y = var_620)[name = tensor("op_621")]; + tensor clip_13 = clip(alpha = var_22, beta = var_17, x = var_621)[name = tensor("clip_13")]; + tensor var_624_begin_0 = const()[name = tensor("op_624_begin_0"), val = tensor([0, 2, 0])]; + tensor var_624_end_0 = const()[name = tensor("op_624_end_0"), val = tensor([1, 3, 256])]; + tensor var_624_end_mask_0 = const()[name = tensor("op_624_end_mask_0"), val = tensor([true, false, true])]; + tensor var_624 = slice_by_index(begin = var_624_begin_0, end = var_624_end_0, end_mask = var_624_end_mask_0, x = x_9)[name = tensor("op_624")]; + tensor var_627_begin_0 = const()[name = tensor("op_627_begin_0"), val = tensor([0, 1, 0])]; + tensor var_627_end_0 = const()[name = tensor("op_627_end_0"), val = tensor([1, 16, 256])]; + tensor var_627_end_mask_0 = const()[name = tensor("op_627_end_mask_0"), val = tensor([true, true, true])]; + tensor var_627 = slice_by_index(begin = var_627_begin_0, end = var_627_end_0, end_mask = var_627_end_mask_0, x = window_17)[name = tensor("op_627")]; tensor rolled_15_interleave_0 = const()[name = tensor("rolled_15_interleave_0"), val = tensor(false)]; - tensor rolled_15 = concat(axis = var_34, interleave = rolled_15_interleave_0, values = (var_950, var_947))[name = tensor("rolled_15")]; + tensor rolled_15 = concat(axis = var_34, interleave = rolled_15_interleave_0, values = (var_627, var_624))[name = tensor("rolled_15")]; tensor clip_14 = clip(alpha = var_22, beta = var_21, x = clip_13)[name = tensor("clip_14")]; - tensor var_955 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_955")]; - tensor var_956_dtype_0 = const()[name = tensor("op_956_dtype_0"), val = tensor("fp32")]; - tensor var_957 = const()[name = tensor("op_957"), val = tensor([1, 16, 1])]; - tensor var_956 = cast(dtype = var_956_dtype_0, x = var_955)[name = tensor("cast_177")]; - tensor fill_mask_15 = reshape(shape = var_957, x = var_956)[name = tensor("fill_mask_15")]; + tensor var_632 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_632")]; + tensor var_633_dtype_0 = const()[name = tensor("op_633_dtype_0"), val = tensor("fp32")]; + tensor var_634 = const()[name = tensor("op_634"), val = tensor([1, 16, 1])]; + tensor var_633 = cast(dtype = var_633_dtype_0, x = var_632)[name = tensor("cast_177")]; + tensor fill_mask_15 = reshape(shape = var_634, x = var_633)[name = tensor("fill_mask_15")]; tensor x_expanded_15_reps_0 = const()[name = tensor("x_expanded_15_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_947)[name = tensor("x_expanded_15")]; - tensor var_961 = sub(x = var_18, y = fill_mask_15)[name = tensor("op_961")]; - tensor var_962 = mul(x = input_111, y = var_961)[name = tensor("op_962")]; - tensor var_963 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_963")]; - tensor filled_window_15 = add(x = var_962, y = var_963)[name = tensor("filled_window_15")]; - tensor var_965 = greater_equal(x = clip_13, y = var_17)[name = tensor("op_965")]; - tensor var_966_dtype_0 = const()[name = tensor("op_966_dtype_0"), val = tensor("fp32")]; - tensor var_967 = const()[name = tensor("op_967"), val = tensor([1, 1, 1])]; - tensor var_966 = cast(dtype = var_966_dtype_0, x = var_965)[name = tensor("cast_176")]; - tensor use_roll_15 = reshape(shape = var_967, x = var_966)[name = tensor("use_roll_15")]; - tensor var_969 = sub(x = var_18, y = use_roll_15)[name = tensor("op_969")]; - tensor var_970 = mul(x = filled_window_15, y = var_969)[name = tensor("op_970")]; - tensor var_971 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_971")]; - tensor input_121 = add(x = var_970, y = var_971)[name = tensor("input_121")]; - tensor var_973 = const()[name = tensor("op_973"), val = tensor(0x1p+0)]; - tensor var_974 = add(x = clip_13, y = var_973)[name = tensor("op_974")]; - tensor clip_15 = clip(alpha = var_22, beta = var_17, x = var_974)[name = tensor("clip_15")]; - tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; - tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_121)[name = tensor("x_23")]; - tensor input_123_perm_0 = const()[name = tensor("input_123_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_49_pad_type_0 = const()[name = tensor("inputs_49_pad_type_0"), val = tensor("valid")]; - tensor inputs_49_strides_0 = const()[name = tensor("inputs_49_strides_0"), val = tensor([1])]; - tensor inputs_49_pad_0 = const()[name = tensor("inputs_49_pad_0"), val = tensor([0, 0])]; - tensor inputs_49_dilations_0 = const()[name = tensor("inputs_49_dilations_0"), val = tensor([1])]; - tensor inputs_49_groups_0 = const()[name = tensor("inputs_49_groups_0"), val = tensor(1)]; - tensor input_123 = transpose(perm = input_123_perm_0, x = x_23)[name = tensor("transpose_64")]; - tensor inputs_49 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_49_dilations_0, groups = inputs_49_groups_0, pad = inputs_49_pad_0, pad_type = inputs_49_pad_type_0, strides = inputs_49_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_123)[name = tensor("inputs_49")]; - tensor var_995_split_sizes_0 = const()[name = tensor("op_995_split_sizes_0"), val = tensor([256, 256])]; - tensor var_995_axis_0 = const()[name = tensor("op_995_axis_0"), val = tensor(1)]; - tensor var_995_0, tensor var_995_1 = split(axis = var_995_axis_0, split_sizes = var_995_split_sizes_0, x = inputs_49)[name = tensor("op_995")]; - tensor var_997 = sigmoid(x = var_995_1)[name = tensor("op_997")]; - tensor inputs_51 = mul(x = var_995_0, y = var_997)[name = tensor("inputs_51")]; - tensor outputs_aug_15_pad_type_0 = const()[name = tensor("outputs_aug_15_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_15_pad_0 = const()[name = tensor("outputs_aug_15_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_15_groups_0 = const()[name = tensor("outputs_aug_15_groups_0"), val = tensor(256)]; - tensor outputs_aug_15_strides_0 = const()[name = tensor("outputs_aug_15_strides_0"), val = tensor([1])]; - tensor outputs_aug_15_dilations_0 = const()[name = tensor("outputs_aug_15_dilations_0"), val = tensor([1])]; - tensor outputs_aug_15 = conv(dilations = outputs_aug_15_dilations_0, groups = outputs_aug_15_groups_0, pad = outputs_aug_15_pad_0, pad_type = outputs_aug_15_pad_type_0, strides = outputs_aug_15_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_51)[name = tensor("outputs_aug_15")]; - tensor input_125_begin_0 = const()[name = tensor("input_125_begin_0"), val = tensor([0, 0, 0])]; - tensor input_125_end_0 = const()[name = tensor("input_125_end_0"), val = tensor([1, 256, 16])]; - tensor input_125_end_mask_0 = const()[name = tensor("input_125_end_mask_0"), val = tensor([true, true, false])]; - tensor input_125 = slice_by_index(begin = input_125_begin_0, end = input_125_end_0, end_mask = input_125_end_mask_0, x = outputs_aug_15)[name = tensor("input_125")]; - tensor inputs_53 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_125)[name = tensor("inputs_53")]; - tensor input_127 = silu(x = inputs_53)[name = tensor("input_127")]; - tensor input_129_pad_type_0 = const()[name = tensor("input_129_pad_type_0"), val = tensor("valid")]; - tensor input_129_strides_0 = const()[name = tensor("input_129_strides_0"), val = tensor([1])]; - tensor input_129_pad_0 = const()[name = tensor("input_129_pad_0"), val = tensor([0, 0])]; - tensor input_129_dilations_0 = const()[name = tensor("input_129_dilations_0"), val = tensor([1])]; - tensor input_129_groups_0 = const()[name = tensor("input_129_groups_0"), val = tensor(1)]; - tensor input_129 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_129_dilations_0, groups = input_129_groups_0, pad = input_129_pad_0, pad_type = input_129_pad_type_0, strides = input_129_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_127)[name = tensor("input_129")]; - tensor conv_out_15_perm_0 = const()[name = tensor("conv_out_15_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1028_begin_0 = const()[name = tensor("op_1028_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1028_end_0 = const()[name = tensor("op_1028_end_0"), val = tensor([1, 16, 256])]; - tensor var_1028_end_mask_0 = const()[name = tensor("op_1028_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_15 = transpose(perm = conv_out_15_perm_0, x = input_129)[name = tensor("transpose_63")]; - tensor var_1028 = slice_by_index(begin = var_1028_begin_0, end = var_1028_end_0, end_mask = var_1028_end_mask_0, x = conv_out_15)[name = tensor("op_1028")]; - tensor var_1031_begin_0 = const()[name = tensor("op_1031_begin_0"), val = tensor([0, 3, 0])]; - tensor var_1031_end_0 = const()[name = tensor("op_1031_end_0"), val = tensor([1, 4, 256])]; - tensor var_1031_end_mask_0 = const()[name = tensor("op_1031_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1031 = slice_by_index(begin = var_1031_begin_0, end = var_1031_end_0, end_mask = var_1031_end_mask_0, x = x_17)[name = tensor("op_1031")]; - tensor var_1034_begin_0 = const()[name = tensor("op_1034_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1034_end_0 = const()[name = tensor("op_1034_end_0"), val = tensor([1, 16, 256])]; - tensor var_1034_end_mask_0 = const()[name = tensor("op_1034_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1034 = slice_by_index(begin = var_1034_begin_0, end = var_1034_end_0, end_mask = var_1034_end_mask_0, x = input_121)[name = tensor("op_1034")]; + tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_624)[name = tensor("x_expanded_15")]; + tensor var_638 = sub(x = var_18, y = fill_mask_15)[name = tensor("op_638")]; + tensor var_639 = mul(x = window_17, y = var_638)[name = tensor("op_639")]; + tensor var_640 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_640")]; + tensor filled_window_15 = add(x = var_639, y = var_640)[name = tensor("filled_window_15")]; + tensor var_642 = greater_equal(x = clip_13, y = var_17)[name = tensor("op_642")]; + tensor var_643_dtype_0 = const()[name = tensor("op_643_dtype_0"), val = tensor("fp32")]; + tensor var_644 = const()[name = tensor("op_644"), val = tensor([1, 1, 1])]; + tensor var_643 = cast(dtype = var_643_dtype_0, x = var_642)[name = tensor("cast_176")]; + tensor use_roll_15 = reshape(shape = var_644, x = var_643)[name = tensor("use_roll_15")]; + tensor var_646 = sub(x = var_18, y = use_roll_15)[name = tensor("op_646")]; + tensor var_647 = mul(x = filled_window_15, y = var_646)[name = tensor("op_647")]; + tensor var_648 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_648")]; + tensor window_19 = add(x = var_647, y = var_648)[name = tensor("window_19")]; + tensor var_650 = const()[name = tensor("op_650"), val = tensor(0x1p+0)]; + tensor var_651 = add(x = clip_13, y = var_650)[name = tensor("op_651")]; + tensor clip_15 = clip(alpha = var_22, beta = var_17, x = var_651)[name = tensor("clip_15")]; + tensor var_654_begin_0 = const()[name = tensor("op_654_begin_0"), val = tensor([0, 3, 0])]; + tensor var_654_end_0 = const()[name = tensor("op_654_end_0"), val = tensor([1, 4, 256])]; + tensor var_654_end_mask_0 = const()[name = tensor("op_654_end_mask_0"), val = tensor([true, false, true])]; + tensor var_654 = slice_by_index(begin = var_654_begin_0, end = var_654_end_0, end_mask = var_654_end_mask_0, x = x_9)[name = tensor("op_654")]; + tensor var_657_begin_0 = const()[name = tensor("op_657_begin_0"), val = tensor([0, 1, 0])]; + tensor var_657_end_0 = const()[name = tensor("op_657_end_0"), val = tensor([1, 16, 256])]; + tensor var_657_end_mask_0 = const()[name = tensor("op_657_end_mask_0"), val = tensor([true, true, true])]; + tensor var_657 = slice_by_index(begin = var_657_begin_0, end = var_657_end_0, end_mask = var_657_end_mask_0, x = window_19)[name = tensor("op_657")]; tensor rolled_17_interleave_0 = const()[name = tensor("rolled_17_interleave_0"), val = tensor(false)]; - tensor rolled_17 = concat(axis = var_34, interleave = rolled_17_interleave_0, values = (var_1034, var_1031))[name = tensor("rolled_17")]; + tensor rolled_17 = concat(axis = var_34, interleave = rolled_17_interleave_0, values = (var_657, var_654))[name = tensor("rolled_17")]; tensor clip_16 = clip(alpha = var_22, beta = var_21, x = clip_15)[name = tensor("clip_16")]; - tensor var_1039 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_1039")]; - tensor var_1040_dtype_0 = const()[name = tensor("op_1040_dtype_0"), val = tensor("fp32")]; - tensor var_1041 = const()[name = tensor("op_1041"), val = tensor([1, 16, 1])]; - tensor var_1040 = cast(dtype = var_1040_dtype_0, x = var_1039)[name = tensor("cast_175")]; - tensor fill_mask_17 = reshape(shape = var_1041, x = var_1040)[name = tensor("fill_mask_17")]; + tensor var_662 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_662")]; + tensor var_663_dtype_0 = const()[name = tensor("op_663_dtype_0"), val = tensor("fp32")]; + tensor var_664 = const()[name = tensor("op_664"), val = tensor([1, 16, 1])]; + tensor var_663 = cast(dtype = var_663_dtype_0, x = var_662)[name = tensor("cast_175")]; + tensor fill_mask_17 = reshape(shape = var_664, x = var_663)[name = tensor("fill_mask_17")]; tensor x_expanded_17_reps_0 = const()[name = tensor("x_expanded_17_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_1031)[name = tensor("x_expanded_17")]; - tensor var_1045 = sub(x = var_18, y = fill_mask_17)[name = tensor("op_1045")]; - tensor var_1046 = mul(x = input_121, y = var_1045)[name = tensor("op_1046")]; - tensor var_1047 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_1047")]; - tensor filled_window_17 = add(x = var_1046, y = var_1047)[name = tensor("filled_window_17")]; - tensor var_1049 = greater_equal(x = clip_15, y = var_17)[name = tensor("op_1049")]; - tensor var_1050_dtype_0 = const()[name = tensor("op_1050_dtype_0"), val = tensor("fp32")]; - tensor var_1051 = const()[name = tensor("op_1051"), val = tensor([1, 1, 1])]; - tensor var_1050 = cast(dtype = var_1050_dtype_0, x = var_1049)[name = tensor("cast_174")]; - tensor use_roll_17 = reshape(shape = var_1051, x = var_1050)[name = tensor("use_roll_17")]; - tensor var_1053 = sub(x = var_18, y = use_roll_17)[name = tensor("op_1053")]; - tensor var_1054 = mul(x = filled_window_17, y = var_1053)[name = tensor("op_1054")]; - tensor var_1055 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_1055")]; - tensor input_131 = add(x = var_1054, y = var_1055)[name = tensor("input_131")]; - tensor var_1057 = const()[name = tensor("op_1057"), val = tensor(0x1p+0)]; - tensor var_1058 = add(x = clip_15, y = var_1057)[name = tensor("op_1058")]; - tensor clip_17 = clip(alpha = var_22, beta = var_17, x = var_1058)[name = tensor("clip_17")]; - tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; - tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_131)[name = tensor("x_25")]; - tensor input_133_perm_0 = const()[name = tensor("input_133_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_55_pad_type_0 = const()[name = tensor("inputs_55_pad_type_0"), val = tensor("valid")]; - tensor inputs_55_strides_0 = const()[name = tensor("inputs_55_strides_0"), val = tensor([1])]; - tensor inputs_55_pad_0 = const()[name = tensor("inputs_55_pad_0"), val = tensor([0, 0])]; - tensor inputs_55_dilations_0 = const()[name = tensor("inputs_55_dilations_0"), val = tensor([1])]; - tensor inputs_55_groups_0 = const()[name = tensor("inputs_55_groups_0"), val = tensor(1)]; - tensor input_133 = transpose(perm = input_133_perm_0, x = x_25)[name = tensor("transpose_62")]; - tensor inputs_55 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_55_dilations_0, groups = inputs_55_groups_0, pad = inputs_55_pad_0, pad_type = inputs_55_pad_type_0, strides = inputs_55_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_133)[name = tensor("inputs_55")]; - tensor var_1079_split_sizes_0 = const()[name = tensor("op_1079_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1079_axis_0 = const()[name = tensor("op_1079_axis_0"), val = tensor(1)]; - tensor var_1079_0, tensor var_1079_1 = split(axis = var_1079_axis_0, split_sizes = var_1079_split_sizes_0, x = inputs_55)[name = tensor("op_1079")]; - tensor var_1081 = sigmoid(x = var_1079_1)[name = tensor("op_1081")]; - tensor inputs_57 = mul(x = var_1079_0, y = var_1081)[name = tensor("inputs_57")]; - tensor outputs_aug_17_pad_type_0 = const()[name = tensor("outputs_aug_17_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_17_pad_0 = const()[name = tensor("outputs_aug_17_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_17_groups_0 = const()[name = tensor("outputs_aug_17_groups_0"), val = tensor(256)]; - tensor outputs_aug_17_strides_0 = const()[name = tensor("outputs_aug_17_strides_0"), val = tensor([1])]; - tensor outputs_aug_17_dilations_0 = const()[name = tensor("outputs_aug_17_dilations_0"), val = tensor([1])]; - tensor outputs_aug_17 = conv(dilations = outputs_aug_17_dilations_0, groups = outputs_aug_17_groups_0, pad = outputs_aug_17_pad_0, pad_type = outputs_aug_17_pad_type_0, strides = outputs_aug_17_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_57)[name = tensor("outputs_aug_17")]; - tensor input_135_begin_0 = const()[name = tensor("input_135_begin_0"), val = tensor([0, 0, 0])]; - tensor input_135_end_0 = const()[name = tensor("input_135_end_0"), val = tensor([1, 256, 16])]; - tensor input_135_end_mask_0 = const()[name = tensor("input_135_end_mask_0"), val = tensor([true, true, false])]; - tensor input_135 = slice_by_index(begin = input_135_begin_0, end = input_135_end_0, end_mask = input_135_end_mask_0, x = outputs_aug_17)[name = tensor("input_135")]; - tensor inputs_59 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_135)[name = tensor("inputs_59")]; - tensor input_137 = silu(x = inputs_59)[name = tensor("input_137")]; - tensor input_139_pad_type_0 = const()[name = tensor("input_139_pad_type_0"), val = tensor("valid")]; - tensor input_139_strides_0 = const()[name = tensor("input_139_strides_0"), val = tensor([1])]; - tensor input_139_pad_0 = const()[name = tensor("input_139_pad_0"), val = tensor([0, 0])]; - tensor input_139_dilations_0 = const()[name = tensor("input_139_dilations_0"), val = tensor([1])]; - tensor input_139_groups_0 = const()[name = tensor("input_139_groups_0"), val = tensor(1)]; - tensor input_139 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_139_dilations_0, groups = input_139_groups_0, pad = input_139_pad_0, pad_type = input_139_pad_type_0, strides = input_139_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_137)[name = tensor("input_139")]; - tensor conv_out_17_perm_0 = const()[name = tensor("conv_out_17_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1112_begin_0 = const()[name = tensor("op_1112_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1112_end_0 = const()[name = tensor("op_1112_end_0"), val = tensor([1, 16, 256])]; - tensor var_1112_end_mask_0 = const()[name = tensor("op_1112_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_17 = transpose(perm = conv_out_17_perm_0, x = input_139)[name = tensor("transpose_61")]; - tensor var_1112 = slice_by_index(begin = var_1112_begin_0, end = var_1112_end_0, end_mask = var_1112_end_mask_0, x = conv_out_17)[name = tensor("op_1112")]; - tensor var_1115_begin_0 = const()[name = tensor("op_1115_begin_0"), val = tensor([0, 4, 0])]; - tensor var_1115_end_0 = const()[name = tensor("op_1115_end_0"), val = tensor([1, 1, 256])]; - tensor var_1115_end_mask_0 = const()[name = tensor("op_1115_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1115 = slice_by_index(begin = var_1115_begin_0, end = var_1115_end_0, end_mask = var_1115_end_mask_0, x = x_17)[name = tensor("op_1115")]; - tensor var_1118_begin_0 = const()[name = tensor("op_1118_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1118_end_0 = const()[name = tensor("op_1118_end_0"), val = tensor([1, 16, 256])]; - tensor var_1118_end_mask_0 = const()[name = tensor("op_1118_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1118 = slice_by_index(begin = var_1118_begin_0, end = var_1118_end_0, end_mask = var_1118_end_mask_0, x = input_131)[name = tensor("op_1118")]; + tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_654)[name = tensor("x_expanded_17")]; + tensor var_668 = sub(x = var_18, y = fill_mask_17)[name = tensor("op_668")]; + tensor var_669 = mul(x = window_19, y = var_668)[name = tensor("op_669")]; + tensor var_670 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_670")]; + tensor filled_window_17 = add(x = var_669, y = var_670)[name = tensor("filled_window_17")]; + tensor var_672 = greater_equal(x = clip_15, y = var_17)[name = tensor("op_672")]; + tensor var_673_dtype_0 = const()[name = tensor("op_673_dtype_0"), val = tensor("fp32")]; + tensor var_674 = const()[name = tensor("op_674"), val = tensor([1, 1, 1])]; + tensor var_673 = cast(dtype = var_673_dtype_0, x = var_672)[name = tensor("cast_174")]; + tensor use_roll_17 = reshape(shape = var_674, x = var_673)[name = tensor("use_roll_17")]; + tensor var_676 = sub(x = var_18, y = use_roll_17)[name = tensor("op_676")]; + tensor var_677 = mul(x = filled_window_17, y = var_676)[name = tensor("op_677")]; + tensor var_678 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_678")]; + tensor window_21 = add(x = var_677, y = var_678)[name = tensor("window_21")]; + tensor var_680 = const()[name = tensor("op_680"), val = tensor(0x1p+0)]; + tensor var_681 = add(x = clip_15, y = var_680)[name = tensor("op_681")]; + tensor clip_17 = clip(alpha = var_22, beta = var_17, x = var_681)[name = tensor("clip_17")]; + tensor var_684_begin_0 = const()[name = tensor("op_684_begin_0"), val = tensor([0, 4, 0])]; + tensor var_684_end_0 = const()[name = tensor("op_684_end_0"), val = tensor([1, 1, 256])]; + tensor var_684_end_mask_0 = const()[name = tensor("op_684_end_mask_0"), val = tensor([true, true, true])]; + tensor var_684 = slice_by_index(begin = var_684_begin_0, end = var_684_end_0, end_mask = var_684_end_mask_0, x = x_9)[name = tensor("op_684")]; + tensor var_687_begin_0 = const()[name = tensor("op_687_begin_0"), val = tensor([0, 1, 0])]; + tensor var_687_end_0 = const()[name = tensor("op_687_end_0"), val = tensor([1, 16, 256])]; + tensor var_687_end_mask_0 = const()[name = tensor("op_687_end_mask_0"), val = tensor([true, true, true])]; + tensor var_687 = slice_by_index(begin = var_687_begin_0, end = var_687_end_0, end_mask = var_687_end_mask_0, x = window_21)[name = tensor("op_687")]; tensor rolled_19_interleave_0 = const()[name = tensor("rolled_19_interleave_0"), val = tensor(false)]; - tensor rolled_19 = concat(axis = var_34, interleave = rolled_19_interleave_0, values = (var_1118, var_1115))[name = tensor("rolled_19")]; + tensor rolled_19 = concat(axis = var_34, interleave = rolled_19_interleave_0, values = (var_687, var_684))[name = tensor("rolled_19")]; tensor clip_18 = clip(alpha = var_22, beta = var_21, x = clip_17)[name = tensor("clip_18")]; - tensor var_1123 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_1123")]; - tensor var_1124_dtype_0 = const()[name = tensor("op_1124_dtype_0"), val = tensor("fp32")]; - tensor var_1125 = const()[name = tensor("op_1125"), val = tensor([1, 16, 1])]; - tensor var_1124 = cast(dtype = var_1124_dtype_0, x = var_1123)[name = tensor("cast_173")]; - tensor fill_mask_19 = reshape(shape = var_1125, x = var_1124)[name = tensor("fill_mask_19")]; + tensor var_692 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_692")]; + tensor var_693_dtype_0 = const()[name = tensor("op_693_dtype_0"), val = tensor("fp32")]; + tensor var_694 = const()[name = tensor("op_694"), val = tensor([1, 16, 1])]; + tensor var_693 = cast(dtype = var_693_dtype_0, x = var_692)[name = tensor("cast_173")]; + tensor fill_mask_19 = reshape(shape = var_694, x = var_693)[name = tensor("fill_mask_19")]; tensor x_expanded_19_reps_0 = const()[name = tensor("x_expanded_19_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_1115)[name = tensor("x_expanded_19")]; - tensor var_1129 = sub(x = var_18, y = fill_mask_19)[name = tensor("op_1129")]; - tensor var_1130 = mul(x = input_131, y = var_1129)[name = tensor("op_1130")]; - tensor var_1131 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_1131")]; - tensor filled_window_19 = add(x = var_1130, y = var_1131)[name = tensor("filled_window_19")]; - tensor var_1133 = greater_equal(x = clip_17, y = var_17)[name = tensor("op_1133")]; - tensor var_1134_dtype_0 = const()[name = tensor("op_1134_dtype_0"), val = tensor("fp32")]; - tensor var_1135 = const()[name = tensor("op_1135"), val = tensor([1, 1, 1])]; - tensor var_1134 = cast(dtype = var_1134_dtype_0, x = var_1133)[name = tensor("cast_172")]; - tensor use_roll_19 = reshape(shape = var_1135, x = var_1134)[name = tensor("use_roll_19")]; - tensor var_1137 = sub(x = var_18, y = use_roll_19)[name = tensor("op_1137")]; - tensor var_1138 = mul(x = filled_window_19, y = var_1137)[name = tensor("op_1138")]; - tensor var_1139 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_1139")]; - tensor input_141 = add(x = var_1138, y = var_1139)[name = tensor("input_141")]; - tensor var_1141 = const()[name = tensor("op_1141"), val = tensor(0x1p+0)]; - tensor var_1142 = add(x = clip_17, y = var_1141)[name = tensor("op_1142")]; - tensor clip_19 = clip(alpha = var_22, beta = var_17, x = var_1142)[name = tensor("clip_19")]; - tensor x_27_axes_0 = const()[name = tensor("x_27_axes_0"), val = tensor([-1])]; - tensor x_27 = layer_norm(axes = x_27_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_141)[name = tensor("x_27")]; - tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_61_pad_type_0 = const()[name = tensor("inputs_61_pad_type_0"), val = tensor("valid")]; - tensor inputs_61_strides_0 = const()[name = tensor("inputs_61_strides_0"), val = tensor([1])]; - tensor inputs_61_pad_0 = const()[name = tensor("inputs_61_pad_0"), val = tensor([0, 0])]; - tensor inputs_61_dilations_0 = const()[name = tensor("inputs_61_dilations_0"), val = tensor([1])]; - tensor inputs_61_groups_0 = const()[name = tensor("inputs_61_groups_0"), val = tensor(1)]; - tensor input_143 = transpose(perm = input_143_perm_0, x = x_27)[name = tensor("transpose_60")]; - tensor inputs_61 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_61_dilations_0, groups = inputs_61_groups_0, pad = inputs_61_pad_0, pad_type = inputs_61_pad_type_0, strides = inputs_61_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_61")]; - tensor var_1163_split_sizes_0 = const()[name = tensor("op_1163_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1163_axis_0 = const()[name = tensor("op_1163_axis_0"), val = tensor(1)]; - tensor var_1163_0, tensor var_1163_1 = split(axis = var_1163_axis_0, split_sizes = var_1163_split_sizes_0, x = inputs_61)[name = tensor("op_1163")]; - tensor var_1165 = sigmoid(x = var_1163_1)[name = tensor("op_1165")]; - tensor inputs_63 = mul(x = var_1163_0, y = var_1165)[name = tensor("inputs_63")]; - tensor outputs_aug_19_pad_type_0 = const()[name = tensor("outputs_aug_19_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_19_pad_0 = const()[name = tensor("outputs_aug_19_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_19_groups_0 = const()[name = tensor("outputs_aug_19_groups_0"), val = tensor(256)]; - tensor outputs_aug_19_strides_0 = const()[name = tensor("outputs_aug_19_strides_0"), val = tensor([1])]; - tensor outputs_aug_19_dilations_0 = const()[name = tensor("outputs_aug_19_dilations_0"), val = tensor([1])]; - tensor outputs_aug_19 = conv(dilations = outputs_aug_19_dilations_0, groups = outputs_aug_19_groups_0, pad = outputs_aug_19_pad_0, pad_type = outputs_aug_19_pad_type_0, strides = outputs_aug_19_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_63)[name = tensor("outputs_aug_19")]; - tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; - tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([1, 256, 16])]; - tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; - tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug_19)[name = tensor("input_145")]; - tensor inputs_65 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_145)[name = tensor("inputs_65")]; - tensor input_147 = silu(x = inputs_65)[name = tensor("input_147")]; - tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; - tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; - tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; - tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; - tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; - tensor input_149 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; - tensor conv_out_19_perm_0 = const()[name = tensor("conv_out_19_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1196_begin_0 = const()[name = tensor("op_1196_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1196_end_0 = const()[name = tensor("op_1196_end_0"), val = tensor([1, 16, 256])]; - tensor var_1196_end_mask_0 = const()[name = tensor("op_1196_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_19 = transpose(perm = conv_out_19_perm_0, x = input_149)[name = tensor("transpose_59")]; - tensor var_1196 = slice_by_index(begin = var_1196_begin_0, end = var_1196_end_0, end_mask = var_1196_end_mask_0, x = conv_out_19)[name = tensor("op_1196")]; - tensor var_1199_interleave_0 = const()[name = tensor("op_1199_interleave_0"), val = tensor(false)]; - tensor var_1199 = concat(axis = var_34, interleave = var_1199_interleave_0, values = (var_860, var_944, var_1028, var_1112, var_1196))[name = tensor("op_1199")]; - tensor input_151 = add(x = x_17, y = var_1199)[name = tensor("input_151")]; - tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; - tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; - tensor inputs_67 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_17")]; - tensor input_155 = silu(x = inputs_67)[name = tensor("input_155")]; - tensor input_159 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_18")]; - tensor var_1222 = const()[name = tensor("op_1222"), val = tensor(0x1p-1)]; - tensor var_1223 = mul(x = input_159, y = var_1222)[name = tensor("op_1223")]; - tensor input_161 = add(x = var_1223, y = input_151)[name = tensor("input_161")]; - tensor input_163_axes_0 = const()[name = tensor("input_163_axes_0"), val = tensor([-1])]; - tensor input_163 = layer_norm(axes = input_163_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_36, gamma = encoder_layer_norm_1_weight, x = input_161)[name = tensor("input_163")]; - tensor input_165_axes_0 = const()[name = tensor("input_165_axes_0"), val = tensor([-1])]; - tensor input_165 = layer_norm(axes = input_165_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_163)[name = tensor("input_165")]; - tensor inputs_69 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_165)[name = tensor("linear_19")]; - tensor input_167 = silu(x = inputs_69)[name = tensor("input_167")]; - tensor input_171 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_167)[name = tensor("linear_20")]; - tensor var_1252 = const()[name = tensor("op_1252"), val = tensor(0x1p-1)]; - tensor var_1253 = mul(x = input_171, y = var_1252)[name = tensor("op_1253")]; - tensor input_173 = add(x = var_1253, y = input_163)[name = tensor("input_173")]; - tensor x_29_axes_0 = const()[name = tensor("x_29_axes_0"), val = tensor([-1])]; - tensor x_29 = layer_norm(axes = x_29_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_36, gamma = encoder_ret_lns_2_weight, x = input_173)[name = tensor("x_29")]; + tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_684)[name = tensor("x_expanded_19")]; + tensor var_698 = sub(x = var_18, y = fill_mask_19)[name = tensor("op_698")]; + tensor var_699 = mul(x = window_21, y = var_698)[name = tensor("op_699")]; + tensor var_700 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_700")]; + tensor filled_window_19 = add(x = var_699, y = var_700)[name = tensor("filled_window_19")]; + tensor var_702 = greater_equal(x = clip_17, y = var_17)[name = tensor("op_702")]; + tensor var_703_dtype_0 = const()[name = tensor("op_703_dtype_0"), val = tensor("fp32")]; + tensor var_704 = const()[name = tensor("op_704"), val = tensor([1, 1, 1])]; + tensor var_703 = cast(dtype = var_703_dtype_0, x = var_702)[name = tensor("cast_172")]; + tensor use_roll_19 = reshape(shape = var_704, x = var_703)[name = tensor("use_roll_19")]; + tensor var_706 = sub(x = var_18, y = use_roll_19)[name = tensor("op_706")]; + tensor var_707 = mul(x = filled_window_19, y = var_706)[name = tensor("op_707")]; + tensor var_708 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_708")]; + tensor window_23 = add(x = var_707, y = var_708)[name = tensor("window_23")]; + tensor var_710 = const()[name = tensor("op_710"), val = tensor(0x1p+0)]; + tensor var_711 = add(x = clip_17, y = var_710)[name = tensor("op_711")]; + tensor clip_19 = clip(alpha = var_22, beta = var_17, x = var_711)[name = tensor("clip_19")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_31, interleave = input_61_interleave_0, values = (window_15, window_17, window_19, window_21, window_23))[name = tensor("input_61")]; + tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; + tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; + tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_13_pad_type_0 = const()[name = tensor("inputs_13_pad_type_0"), val = tensor("valid")]; + tensor inputs_13_strides_0 = const()[name = tensor("inputs_13_strides_0"), val = tensor([1])]; + tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; + tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; + tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; + tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; + tensor var_734_split_sizes_0 = const()[name = tensor("op_734_split_sizes_0"), val = tensor([256, 256])]; + tensor var_734_axis_0 = const()[name = tensor("op_734_axis_0"), val = tensor(1)]; + tensor var_734_0, tensor var_734_1 = split(axis = var_734_axis_0, split_sizes = var_734_split_sizes_0, x = inputs_13)[name = tensor("op_734")]; + tensor var_736 = sigmoid(x = var_734_1)[name = tensor("op_736")]; + tensor inputs_15 = mul(x = var_734_0, y = var_736)[name = tensor("inputs_15")]; + tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; + tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; + tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; + tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_15)[name = tensor("outputs_aug_3")]; + tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; + tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([5, 256, 16])]; + tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; + tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_3)[name = tensor("input_65")]; + tensor inputs_17 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_65)[name = tensor("inputs_17")]; + tensor input_67 = silu(x = inputs_17)[name = tensor("input_67")]; + tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; + tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; + tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; + tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; + tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; + tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; + tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; + tensor var_767_begin_0 = const()[name = tensor("op_767_begin_0"), val = tensor([0, -1, 0])]; + tensor var_767_end_0 = const()[name = tensor("op_767_end_0"), val = tensor([5, 16, 256])]; + tensor var_767_end_mask_0 = const()[name = tensor("op_767_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_767 = slice_by_index(begin = var_767_begin_0, end = var_767_end_0, end_mask = var_767_end_mask_0, x = conv_out_3)[name = tensor("op_767")]; + tensor var_769_perm_0 = const()[name = tensor("op_769_perm_0"), val = tensor([1, 0, 2])]; + tensor var_769 = transpose(perm = var_769_perm_0, x = var_767)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_769)[name = tensor("input_71")]; + tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; + tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; + tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; + tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; + tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; + tensor var_792 = const()[name = tensor("op_792"), val = tensor(0x1p-1)]; + tensor var_793 = mul(x = input_79, y = var_792)[name = tensor("op_793")]; + tensor input_81 = add(x = var_793, y = input_71)[name = tensor("input_81")]; + tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; + tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_36, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; + tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; + tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; + tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; + tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; + tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; + tensor var_822 = const()[name = tensor("op_822"), val = tensor(0x1p-1)]; + tensor var_823 = mul(x = input_91, y = var_822)[name = tensor("op_823")]; + tensor input_93 = add(x = var_823, y = input_83)[name = tensor("input_93")]; + tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; + tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_36, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; tensor prev_kv_5_end_0 = const()[name = tensor("prev_kv_5_end_0"), val = tensor([3, 1, 4, 64, 64])]; tensor prev_kv_5_end_mask_0 = const()[name = tensor("prev_kv_5_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1175,462 +868,308 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_1267 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_29)[name = tensor("linear_21")]; - tensor var_1268 = const()[name = tensor("op_1268"), val = tensor([1, 5, 4, 64])]; - tensor var_1269 = reshape(shape = var_1268, x = var_1267)[name = tensor("op_1269")]; + tensor var_837 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_838 = const()[name = tensor("op_838"), val = tensor([1, 5, 4, 64])]; + tensor var_839 = reshape(shape = var_838, x = var_837)[name = tensor("op_839")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1273 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_29)[name = tensor("linear_22")]; - tensor var_1274 = const()[name = tensor("op_1274"), val = tensor(0x1p-3)]; - tensor var_1275 = mul(x = var_1273, y = var_1274)[name = tensor("op_1275")]; - tensor var_1276 = const()[name = tensor("op_1276"), val = tensor([1, 5, 4, 64])]; - tensor var_1277 = reshape(shape = var_1276, x = var_1275)[name = tensor("op_1277")]; + tensor var_843 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_844 = const()[name = tensor("op_844"), val = tensor(0x1p-3)]; + tensor var_845 = mul(x = var_843, y = var_844)[name = tensor("op_845")]; + tensor var_846 = const()[name = tensor("op_846"), val = tensor([1, 5, 4, 64])]; + tensor var_847 = reshape(shape = var_846, x = var_845)[name = tensor("op_847")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1281 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_29)[name = tensor("linear_23")]; - tensor var_1282 = const()[name = tensor("op_1282"), val = tensor([1, 5, 4, 64])]; - tensor var_1283 = reshape(shape = var_1282, x = var_1281)[name = tensor("op_1283")]; + tensor var_851 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_852 = const()[name = tensor("op_852"), val = tensor([1, 5, 4, 64])]; + tensor var_853 = reshape(shape = var_852, x = var_851)[name = tensor("op_853")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_177 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_29)[name = tensor("linear_24")]; + tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; tensor s_t_5 = add(x = prev_scale_5, y = encoder__t_index)[name = tensor("s_t_5")]; tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_1277)[name = tensor("transpose_57")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_1269)[name = tensor("transpose_58")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_847)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_839)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_1293 = const()[name = tensor("op_1293"), val = tensor([5, 1])]; - tensor var_1294 = reshape(shape = var_1293, x = sqrt_s_t_5)[name = tensor("op_1294")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_1294)[name = tensor("M_5")]; - tensor var_1296 = mul(x = qk_5, y = M_5)[name = tensor("op_1296")]; + tensor var_863 = const()[name = tensor("op_863"), val = tensor([5, 1])]; + tensor var_864 = reshape(shape = var_863, x = sqrt_s_t_5)[name = tensor("op_864")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_864)[name = tensor("M_5")]; + tensor var_866 = mul(x = qk_5, y = M_5)[name = tensor("op_866")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_1283)[name = tensor("transpose_56")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_1296, y = v_5)[name = tensor("inner_5")]; - tensor var_1298_transpose_x_0 = const()[name = tensor("op_1298_transpose_x_0"), val = tensor(false)]; - tensor var_1298_transpose_y_0 = const()[name = tensor("op_1298_transpose_y_0"), val = tensor(false)]; - tensor var_1298 = matmul(transpose_x = var_1298_transpose_x_0, transpose_y = var_1298_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_1298")]; - tensor var_1299 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_1299")]; - tensor var_1300 = const()[name = tensor("op_1300"), val = tensor([1, 1, 5, 1])]; - tensor var_1301 = reshape(shape = var_1300, x = var_1299)[name = tensor("op_1301")]; - tensor cross_5 = mul(x = var_1298, y = var_1301)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_853)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_866, y = v_5)[name = tensor("inner_5")]; + tensor var_868_transpose_x_0 = const()[name = tensor("op_868_transpose_x_0"), val = tensor(false)]; + tensor var_868_transpose_y_0 = const()[name = tensor("op_868_transpose_y_0"), val = tensor(false)]; + tensor var_868 = matmul(transpose_x = var_868_transpose_x_0, transpose_y = var_868_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_868")]; + tensor var_869 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_869")]; + tensor var_870 = const()[name = tensor("op_870"), val = tensor([1, 1, 5, 1])]; + tensor var_871 = reshape(shape = var_870, x = var_869)[name = tensor("op_871")]; + tensor cross_5 = mul(x = var_868, y = var_871)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_1304 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_1304")]; - tensor var_1306_transpose_x_1 = const()[name = tensor("op_1306_transpose_x_1"), val = tensor(true)]; - tensor var_1306_transpose_y_1 = const()[name = tensor("op_1306_transpose_y_1"), val = tensor(false)]; - tensor var_1306 = matmul(transpose_x = var_1306_transpose_x_1, transpose_y = var_1306_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_1306")]; - tensor new_kv_unnorm_5 = add(x = var_1304, y = var_1306)[name = tensor("new_kv_unnorm_5")]; - tensor var_1308 = const()[name = tensor("op_1308"), val = tensor(0x1.4p+2)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_1308)[name = tensor("new_scale_5")]; - tensor var_1310 = sqrt(x = new_scale_5)[name = tensor("op_1310")]; - tensor var_1311 = real_div(x = new_kv_unnorm_5, y = var_1310)[name = tensor("op_1311")]; - tensor var_1312_perm_0 = const()[name = tensor("op_1312_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_874 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_874")]; + tensor var_876_transpose_x_1 = const()[name = tensor("op_876_transpose_x_1"), val = tensor(true)]; + tensor var_876_transpose_y_1 = const()[name = tensor("op_876_transpose_y_1"), val = tensor(false)]; + tensor var_876 = matmul(transpose_x = var_876_transpose_x_1, transpose_y = var_876_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_876")]; + tensor new_kv_unnorm_5 = add(x = var_874, y = var_876)[name = tensor("new_kv_unnorm_5")]; + tensor var_878 = const()[name = tensor("op_878"), val = tensor(0x1.4p+2)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_878)[name = tensor("new_scale_5")]; + tensor var_880 = sqrt(x = new_scale_5)[name = tensor("op_880")]; + tensor var_881 = real_div(x = new_kv_unnorm_5, y = var_880)[name = tensor("op_881")]; + tensor var_882_perm_0 = const()[name = tensor("op_882_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_1312 = transpose(perm = var_1312_perm_0, x = out_13)[name = tensor("transpose_55")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_25, x = var_1312)[name = tensor("out_15")]; - tensor var_1316 = const()[name = tensor("op_1316"), val = tensor([1, 5, 256])]; - tensor out_17 = reshape(shape = var_1316, x = out_15)[name = tensor("out_17")]; - tensor var_1318 = silu(x = input_177)[name = tensor("op_1318")]; - tensor input_179 = mul(x = var_1318, y = out_17)[name = tensor("input_179")]; - tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_179)[name = tensor("linear_25")]; - tensor x_31 = add(x = input_173, y = ret_out_5)[name = tensor("x_31")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor var_882 = transpose(perm = var_882_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_25, x = var_882)[name = tensor("out_15")]; + tensor var_886 = const()[name = tensor("op_886"), val = tensor([1, 5, 256])]; + tensor out_17 = reshape(shape = var_886, x = out_15)[name = tensor("out_17")]; + tensor var_888 = silu(x = input_97)[name = tensor("op_888")]; + tensor input_99 = mul(x = var_888, y = out_17)[name = tensor("input_99")]; + tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; + tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; + tensor window_25_begin_0 = const()[name = tensor("window_25_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_25_end_0 = const()[name = tensor("window_25_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_25_end_mask_0 = const()[name = tensor("window_25_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_25_squeeze_mask_0 = const()[name = tensor("window_25_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_25 = slice_by_index(begin = window_25_begin_0, end = window_25_end_0, end_mask = window_25_end_mask_0, squeeze_mask = window_25_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_25")]; tensor filled_21_begin_0 = const()[name = tensor("filled_21_begin_0"), val = tensor([2, 0])]; tensor filled_21_end_0 = const()[name = tensor("filled_21_end_0"), val = tensor([3, 1])]; tensor filled_21_end_mask_0 = const()[name = tensor("filled_21_end_mask_0"), val = tensor([false, true])]; tensor filled_21_squeeze_mask_0 = const()[name = tensor("filled_21_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_21 = slice_by_index(begin = filled_21_begin_0, end = filled_21_end_0, end_mask = filled_21_end_mask_0, squeeze_mask = filled_21_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_21")]; - tensor var_1327_begin_0 = const()[name = tensor("op_1327_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1327_end_0 = const()[name = tensor("op_1327_end_0"), val = tensor([1, 1, 256])]; - tensor var_1327_end_mask_0 = const()[name = tensor("op_1327_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1327 = slice_by_index(begin = var_1327_begin_0, end = var_1327_end_0, end_mask = var_1327_end_mask_0, x = x_31)[name = tensor("op_1327")]; - tensor var_1330_begin_0 = const()[name = tensor("op_1330_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1330_end_0 = const()[name = tensor("op_1330_end_0"), val = tensor([1, 16, 256])]; - tensor var_1330_end_mask_0 = const()[name = tensor("op_1330_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1330 = slice_by_index(begin = var_1330_begin_0, end = var_1330_end_0, end_mask = var_1330_end_mask_0, x = window_5)[name = tensor("op_1330")]; + tensor var_897_begin_0 = const()[name = tensor("op_897_begin_0"), val = tensor([0, 0, 0])]; + tensor var_897_end_0 = const()[name = tensor("op_897_end_0"), val = tensor([1, 1, 256])]; + tensor var_897_end_mask_0 = const()[name = tensor("op_897_end_mask_0"), val = tensor([true, false, true])]; + tensor var_897 = slice_by_index(begin = var_897_begin_0, end = var_897_end_0, end_mask = var_897_end_mask_0, x = x_15)[name = tensor("op_897")]; + tensor var_900_begin_0 = const()[name = tensor("op_900_begin_0"), val = tensor([0, 1, 0])]; + tensor var_900_end_0 = const()[name = tensor("op_900_end_0"), val = tensor([1, 16, 256])]; + tensor var_900_end_mask_0 = const()[name = tensor("op_900_end_mask_0"), val = tensor([true, true, true])]; + tensor var_900 = slice_by_index(begin = var_900_begin_0, end = var_900_end_0, end_mask = var_900_end_mask_0, x = window_25)[name = tensor("op_900")]; tensor rolled_21_interleave_0 = const()[name = tensor("rolled_21_interleave_0"), val = tensor(false)]; - tensor rolled_21 = concat(axis = var_34, interleave = rolled_21_interleave_0, values = (var_1330, var_1327))[name = tensor("rolled_21")]; + tensor rolled_21 = concat(axis = var_34, interleave = rolled_21_interleave_0, values = (var_900, var_897))[name = tensor("rolled_21")]; tensor clip_20 = clip(alpha = var_22, beta = var_21, x = filled_21)[name = tensor("clip_20")]; - tensor var_1335 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_1335")]; - tensor var_1336_dtype_0 = const()[name = tensor("op_1336_dtype_0"), val = tensor("fp32")]; - tensor var_1337 = const()[name = tensor("op_1337"), val = tensor([1, 16, 1])]; - tensor var_1336 = cast(dtype = var_1336_dtype_0, x = var_1335)[name = tensor("cast_171")]; - tensor fill_mask_21 = reshape(shape = var_1337, x = var_1336)[name = tensor("fill_mask_21")]; + tensor var_905 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_905")]; + tensor var_906_dtype_0 = const()[name = tensor("op_906_dtype_0"), val = tensor("fp32")]; + tensor var_907 = const()[name = tensor("op_907"), val = tensor([1, 16, 1])]; + tensor var_906 = cast(dtype = var_906_dtype_0, x = var_905)[name = tensor("cast_171")]; + tensor fill_mask_21 = reshape(shape = var_907, x = var_906)[name = tensor("fill_mask_21")]; tensor x_expanded_21_reps_0 = const()[name = tensor("x_expanded_21_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_1327)[name = tensor("x_expanded_21")]; - tensor var_1341 = sub(x = var_18, y = fill_mask_21)[name = tensor("op_1341")]; - tensor var_1342 = mul(x = window_5, y = var_1341)[name = tensor("op_1342")]; - tensor var_1343 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_1343")]; - tensor filled_window_21 = add(x = var_1342, y = var_1343)[name = tensor("filled_window_21")]; - tensor var_1345 = greater_equal(x = filled_21, y = var_17)[name = tensor("op_1345")]; - tensor var_1346_dtype_0 = const()[name = tensor("op_1346_dtype_0"), val = tensor("fp32")]; - tensor var_1347 = const()[name = tensor("op_1347"), val = tensor([1, 1, 1])]; - tensor var_1346 = cast(dtype = var_1346_dtype_0, x = var_1345)[name = tensor("cast_170")]; - tensor use_roll_21 = reshape(shape = var_1347, x = var_1346)[name = tensor("use_roll_21")]; - tensor var_1349 = sub(x = var_18, y = use_roll_21)[name = tensor("op_1349")]; - tensor var_1350 = mul(x = filled_window_21, y = var_1349)[name = tensor("op_1350")]; - tensor var_1351 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_1351")]; - tensor input_181 = add(x = var_1350, y = var_1351)[name = tensor("input_181")]; - tensor var_1353 = const()[name = tensor("op_1353"), val = tensor(0x1p+0)]; - tensor var_1354 = add(x = filled_21, y = var_1353)[name = tensor("op_1354")]; - tensor clip_21 = clip(alpha = var_22, beta = var_17, x = var_1354)[name = tensor("clip_21")]; - tensor x_33_axes_0 = const()[name = tensor("x_33_axes_0"), val = tensor([-1])]; - tensor x_33 = layer_norm(axes = x_33_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_181)[name = tensor("x_33")]; - tensor input_183_perm_0 = const()[name = tensor("input_183_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_71_pad_type_0 = const()[name = tensor("inputs_71_pad_type_0"), val = tensor("valid")]; - tensor inputs_71_strides_0 = const()[name = tensor("inputs_71_strides_0"), val = tensor([1])]; - tensor inputs_71_pad_0 = const()[name = tensor("inputs_71_pad_0"), val = tensor([0, 0])]; - tensor inputs_71_dilations_0 = const()[name = tensor("inputs_71_dilations_0"), val = tensor([1])]; - tensor inputs_71_groups_0 = const()[name = tensor("inputs_71_groups_0"), val = tensor(1)]; - tensor input_183 = transpose(perm = input_183_perm_0, x = x_33)[name = tensor("transpose_54")]; - tensor inputs_71 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_71_dilations_0, groups = inputs_71_groups_0, pad = inputs_71_pad_0, pad_type = inputs_71_pad_type_0, strides = inputs_71_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_183)[name = tensor("inputs_71")]; - tensor var_1375_split_sizes_0 = const()[name = tensor("op_1375_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1375_axis_0 = const()[name = tensor("op_1375_axis_0"), val = tensor(1)]; - tensor var_1375_0, tensor var_1375_1 = split(axis = var_1375_axis_0, split_sizes = var_1375_split_sizes_0, x = inputs_71)[name = tensor("op_1375")]; - tensor var_1377 = sigmoid(x = var_1375_1)[name = tensor("op_1377")]; - tensor inputs_73 = mul(x = var_1375_0, y = var_1377)[name = tensor("inputs_73")]; - tensor outputs_aug_21_pad_type_0 = const()[name = tensor("outputs_aug_21_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_21_pad_0 = const()[name = tensor("outputs_aug_21_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_21_groups_0 = const()[name = tensor("outputs_aug_21_groups_0"), val = tensor(256)]; - tensor outputs_aug_21_strides_0 = const()[name = tensor("outputs_aug_21_strides_0"), val = tensor([1])]; - tensor outputs_aug_21_dilations_0 = const()[name = tensor("outputs_aug_21_dilations_0"), val = tensor([1])]; - tensor outputs_aug_21 = conv(dilations = outputs_aug_21_dilations_0, groups = outputs_aug_21_groups_0, pad = outputs_aug_21_pad_0, pad_type = outputs_aug_21_pad_type_0, strides = outputs_aug_21_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_73)[name = tensor("outputs_aug_21")]; - tensor input_185_begin_0 = const()[name = tensor("input_185_begin_0"), val = tensor([0, 0, 0])]; - tensor input_185_end_0 = const()[name = tensor("input_185_end_0"), val = tensor([1, 256, 16])]; - tensor input_185_end_mask_0 = const()[name = tensor("input_185_end_mask_0"), val = tensor([true, true, false])]; - tensor input_185 = slice_by_index(begin = input_185_begin_0, end = input_185_end_0, end_mask = input_185_end_mask_0, x = outputs_aug_21)[name = tensor("input_185")]; - tensor inputs_75 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_185)[name = tensor("inputs_75")]; - tensor input_187 = silu(x = inputs_75)[name = tensor("input_187")]; - tensor input_189_pad_type_0 = const()[name = tensor("input_189_pad_type_0"), val = tensor("valid")]; - tensor input_189_strides_0 = const()[name = tensor("input_189_strides_0"), val = tensor([1])]; - tensor input_189_pad_0 = const()[name = tensor("input_189_pad_0"), val = tensor([0, 0])]; - tensor input_189_dilations_0 = const()[name = tensor("input_189_dilations_0"), val = tensor([1])]; - tensor input_189_groups_0 = const()[name = tensor("input_189_groups_0"), val = tensor(1)]; - tensor input_189 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_189_dilations_0, groups = input_189_groups_0, pad = input_189_pad_0, pad_type = input_189_pad_type_0, strides = input_189_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_187)[name = tensor("input_189")]; - tensor conv_out_21_perm_0 = const()[name = tensor("conv_out_21_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1408_begin_0 = const()[name = tensor("op_1408_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1408_end_0 = const()[name = tensor("op_1408_end_0"), val = tensor([1, 16, 256])]; - tensor var_1408_end_mask_0 = const()[name = tensor("op_1408_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_21 = transpose(perm = conv_out_21_perm_0, x = input_189)[name = tensor("transpose_53")]; - tensor var_1408 = slice_by_index(begin = var_1408_begin_0, end = var_1408_end_0, end_mask = var_1408_end_mask_0, x = conv_out_21)[name = tensor("op_1408")]; - tensor var_1411_begin_0 = const()[name = tensor("op_1411_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1411_end_0 = const()[name = tensor("op_1411_end_0"), val = tensor([1, 2, 256])]; - tensor var_1411_end_mask_0 = const()[name = tensor("op_1411_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1411 = slice_by_index(begin = var_1411_begin_0, end = var_1411_end_0, end_mask = var_1411_end_mask_0, x = x_31)[name = tensor("op_1411")]; - tensor var_1414_begin_0 = const()[name = tensor("op_1414_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1414_end_0 = const()[name = tensor("op_1414_end_0"), val = tensor([1, 16, 256])]; - tensor var_1414_end_mask_0 = const()[name = tensor("op_1414_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1414 = slice_by_index(begin = var_1414_begin_0, end = var_1414_end_0, end_mask = var_1414_end_mask_0, x = input_181)[name = tensor("op_1414")]; + tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_897)[name = tensor("x_expanded_21")]; + tensor var_911 = sub(x = var_18, y = fill_mask_21)[name = tensor("op_911")]; + tensor var_912 = mul(x = window_25, y = var_911)[name = tensor("op_912")]; + tensor var_913 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_913")]; + tensor filled_window_21 = add(x = var_912, y = var_913)[name = tensor("filled_window_21")]; + tensor var_915 = greater_equal(x = filled_21, y = var_17)[name = tensor("op_915")]; + tensor var_916_dtype_0 = const()[name = tensor("op_916_dtype_0"), val = tensor("fp32")]; + tensor var_917 = const()[name = tensor("op_917"), val = tensor([1, 1, 1])]; + tensor var_916 = cast(dtype = var_916_dtype_0, x = var_915)[name = tensor("cast_170")]; + tensor use_roll_21 = reshape(shape = var_917, x = var_916)[name = tensor("use_roll_21")]; + tensor var_919 = sub(x = var_18, y = use_roll_21)[name = tensor("op_919")]; + tensor var_920 = mul(x = filled_window_21, y = var_919)[name = tensor("op_920")]; + tensor var_921 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_921")]; + tensor window_27 = add(x = var_920, y = var_921)[name = tensor("window_27")]; + tensor var_923 = const()[name = tensor("op_923"), val = tensor(0x1p+0)]; + tensor var_924 = add(x = filled_21, y = var_923)[name = tensor("op_924")]; + tensor clip_21 = clip(alpha = var_22, beta = var_17, x = var_924)[name = tensor("clip_21")]; + tensor var_927_begin_0 = const()[name = tensor("op_927_begin_0"), val = tensor([0, 1, 0])]; + tensor var_927_end_0 = const()[name = tensor("op_927_end_0"), val = tensor([1, 2, 256])]; + tensor var_927_end_mask_0 = const()[name = tensor("op_927_end_mask_0"), val = tensor([true, false, true])]; + tensor var_927 = slice_by_index(begin = var_927_begin_0, end = var_927_end_0, end_mask = var_927_end_mask_0, x = x_15)[name = tensor("op_927")]; + tensor var_930_begin_0 = const()[name = tensor("op_930_begin_0"), val = tensor([0, 1, 0])]; + tensor var_930_end_0 = const()[name = tensor("op_930_end_0"), val = tensor([1, 16, 256])]; + tensor var_930_end_mask_0 = const()[name = tensor("op_930_end_mask_0"), val = tensor([true, true, true])]; + tensor var_930 = slice_by_index(begin = var_930_begin_0, end = var_930_end_0, end_mask = var_930_end_mask_0, x = window_27)[name = tensor("op_930")]; tensor rolled_23_interleave_0 = const()[name = tensor("rolled_23_interleave_0"), val = tensor(false)]; - tensor rolled_23 = concat(axis = var_34, interleave = rolled_23_interleave_0, values = (var_1414, var_1411))[name = tensor("rolled_23")]; + tensor rolled_23 = concat(axis = var_34, interleave = rolled_23_interleave_0, values = (var_930, var_927))[name = tensor("rolled_23")]; tensor clip_22 = clip(alpha = var_22, beta = var_21, x = clip_21)[name = tensor("clip_22")]; - tensor var_1419 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_1419")]; - tensor var_1420_dtype_0 = const()[name = tensor("op_1420_dtype_0"), val = tensor("fp32")]; - tensor var_1421 = const()[name = tensor("op_1421"), val = tensor([1, 16, 1])]; - tensor var_1420 = cast(dtype = var_1420_dtype_0, x = var_1419)[name = tensor("cast_169")]; - tensor fill_mask_23 = reshape(shape = var_1421, x = var_1420)[name = tensor("fill_mask_23")]; + tensor var_935 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_935")]; + tensor var_936_dtype_0 = const()[name = tensor("op_936_dtype_0"), val = tensor("fp32")]; + tensor var_937 = const()[name = tensor("op_937"), val = tensor([1, 16, 1])]; + tensor var_936 = cast(dtype = var_936_dtype_0, x = var_935)[name = tensor("cast_169")]; + tensor fill_mask_23 = reshape(shape = var_937, x = var_936)[name = tensor("fill_mask_23")]; tensor x_expanded_23_reps_0 = const()[name = tensor("x_expanded_23_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_23 = tile(reps = x_expanded_23_reps_0, x = var_1411)[name = tensor("x_expanded_23")]; - tensor var_1425 = sub(x = var_18, y = fill_mask_23)[name = tensor("op_1425")]; - tensor var_1426 = mul(x = input_181, y = var_1425)[name = tensor("op_1426")]; - tensor var_1427 = mul(x = x_expanded_23, y = fill_mask_23)[name = tensor("op_1427")]; - tensor filled_window_23 = add(x = var_1426, y = var_1427)[name = tensor("filled_window_23")]; - tensor var_1429 = greater_equal(x = clip_21, y = var_17)[name = tensor("op_1429")]; - tensor var_1430_dtype_0 = const()[name = tensor("op_1430_dtype_0"), val = tensor("fp32")]; - tensor var_1431 = const()[name = tensor("op_1431"), val = tensor([1, 1, 1])]; - tensor var_1430 = cast(dtype = var_1430_dtype_0, x = var_1429)[name = tensor("cast_168")]; - tensor use_roll_23 = reshape(shape = var_1431, x = var_1430)[name = tensor("use_roll_23")]; - tensor var_1433 = sub(x = var_18, y = use_roll_23)[name = tensor("op_1433")]; - tensor var_1434 = mul(x = filled_window_23, y = var_1433)[name = tensor("op_1434")]; - tensor var_1435 = mul(x = rolled_23, y = use_roll_23)[name = tensor("op_1435")]; - tensor input_191 = add(x = var_1434, y = var_1435)[name = tensor("input_191")]; - tensor var_1437 = const()[name = tensor("op_1437"), val = tensor(0x1p+0)]; - tensor var_1438 = add(x = clip_21, y = var_1437)[name = tensor("op_1438")]; - tensor clip_23 = clip(alpha = var_22, beta = var_17, x = var_1438)[name = tensor("clip_23")]; - tensor x_35_axes_0 = const()[name = tensor("x_35_axes_0"), val = tensor([-1])]; - tensor x_35 = layer_norm(axes = x_35_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_191)[name = tensor("x_35")]; - tensor input_193_perm_0 = const()[name = tensor("input_193_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_77_pad_type_0 = const()[name = tensor("inputs_77_pad_type_0"), val = tensor("valid")]; - tensor inputs_77_strides_0 = const()[name = tensor("inputs_77_strides_0"), val = tensor([1])]; - tensor inputs_77_pad_0 = const()[name = tensor("inputs_77_pad_0"), val = tensor([0, 0])]; - tensor inputs_77_dilations_0 = const()[name = tensor("inputs_77_dilations_0"), val = tensor([1])]; - tensor inputs_77_groups_0 = const()[name = tensor("inputs_77_groups_0"), val = tensor(1)]; - tensor input_193 = transpose(perm = input_193_perm_0, x = x_35)[name = tensor("transpose_52")]; - tensor inputs_77 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_77_dilations_0, groups = inputs_77_groups_0, pad = inputs_77_pad_0, pad_type = inputs_77_pad_type_0, strides = inputs_77_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_193)[name = tensor("inputs_77")]; - tensor var_1459_split_sizes_0 = const()[name = tensor("op_1459_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1459_axis_0 = const()[name = tensor("op_1459_axis_0"), val = tensor(1)]; - tensor var_1459_0, tensor var_1459_1 = split(axis = var_1459_axis_0, split_sizes = var_1459_split_sizes_0, x = inputs_77)[name = tensor("op_1459")]; - tensor var_1461 = sigmoid(x = var_1459_1)[name = tensor("op_1461")]; - tensor inputs_79 = mul(x = var_1459_0, y = var_1461)[name = tensor("inputs_79")]; - tensor outputs_aug_23_pad_type_0 = const()[name = tensor("outputs_aug_23_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_23_pad_0 = const()[name = tensor("outputs_aug_23_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_23_groups_0 = const()[name = tensor("outputs_aug_23_groups_0"), val = tensor(256)]; - tensor outputs_aug_23_strides_0 = const()[name = tensor("outputs_aug_23_strides_0"), val = tensor([1])]; - tensor outputs_aug_23_dilations_0 = const()[name = tensor("outputs_aug_23_dilations_0"), val = tensor([1])]; - tensor outputs_aug_23 = conv(dilations = outputs_aug_23_dilations_0, groups = outputs_aug_23_groups_0, pad = outputs_aug_23_pad_0, pad_type = outputs_aug_23_pad_type_0, strides = outputs_aug_23_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_79)[name = tensor("outputs_aug_23")]; - tensor input_195_begin_0 = const()[name = tensor("input_195_begin_0"), val = tensor([0, 0, 0])]; - tensor input_195_end_0 = const()[name = tensor("input_195_end_0"), val = tensor([1, 256, 16])]; - tensor input_195_end_mask_0 = const()[name = tensor("input_195_end_mask_0"), val = tensor([true, true, false])]; - tensor input_195 = slice_by_index(begin = input_195_begin_0, end = input_195_end_0, end_mask = input_195_end_mask_0, x = outputs_aug_23)[name = tensor("input_195")]; - tensor inputs_81 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_195)[name = tensor("inputs_81")]; - tensor input_197 = silu(x = inputs_81)[name = tensor("input_197")]; - tensor input_199_pad_type_0 = const()[name = tensor("input_199_pad_type_0"), val = tensor("valid")]; - tensor input_199_strides_0 = const()[name = tensor("input_199_strides_0"), val = tensor([1])]; - tensor input_199_pad_0 = const()[name = tensor("input_199_pad_0"), val = tensor([0, 0])]; - tensor input_199_dilations_0 = const()[name = tensor("input_199_dilations_0"), val = tensor([1])]; - tensor input_199_groups_0 = const()[name = tensor("input_199_groups_0"), val = tensor(1)]; - tensor input_199 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_199_dilations_0, groups = input_199_groups_0, pad = input_199_pad_0, pad_type = input_199_pad_type_0, strides = input_199_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_197)[name = tensor("input_199")]; - tensor conv_out_23_perm_0 = const()[name = tensor("conv_out_23_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1492_begin_0 = const()[name = tensor("op_1492_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1492_end_0 = const()[name = tensor("op_1492_end_0"), val = tensor([1, 16, 256])]; - tensor var_1492_end_mask_0 = const()[name = tensor("op_1492_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_23 = transpose(perm = conv_out_23_perm_0, x = input_199)[name = tensor("transpose_51")]; - tensor var_1492 = slice_by_index(begin = var_1492_begin_0, end = var_1492_end_0, end_mask = var_1492_end_mask_0, x = conv_out_23)[name = tensor("op_1492")]; - tensor var_1495_begin_0 = const()[name = tensor("op_1495_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1495_end_0 = const()[name = tensor("op_1495_end_0"), val = tensor([1, 3, 256])]; - tensor var_1495_end_mask_0 = const()[name = tensor("op_1495_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1495 = slice_by_index(begin = var_1495_begin_0, end = var_1495_end_0, end_mask = var_1495_end_mask_0, x = x_31)[name = tensor("op_1495")]; - tensor var_1498_begin_0 = const()[name = tensor("op_1498_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1498_end_0 = const()[name = tensor("op_1498_end_0"), val = tensor([1, 16, 256])]; - tensor var_1498_end_mask_0 = const()[name = tensor("op_1498_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1498 = slice_by_index(begin = var_1498_begin_0, end = var_1498_end_0, end_mask = var_1498_end_mask_0, x = input_191)[name = tensor("op_1498")]; + tensor x_expanded_23 = tile(reps = x_expanded_23_reps_0, x = var_927)[name = tensor("x_expanded_23")]; + tensor var_941 = sub(x = var_18, y = fill_mask_23)[name = tensor("op_941")]; + tensor var_942 = mul(x = window_27, y = var_941)[name = tensor("op_942")]; + tensor var_943 = mul(x = x_expanded_23, y = fill_mask_23)[name = tensor("op_943")]; + tensor filled_window_23 = add(x = var_942, y = var_943)[name = tensor("filled_window_23")]; + tensor var_945 = greater_equal(x = clip_21, y = var_17)[name = tensor("op_945")]; + tensor var_946_dtype_0 = const()[name = tensor("op_946_dtype_0"), val = tensor("fp32")]; + tensor var_947 = const()[name = tensor("op_947"), val = tensor([1, 1, 1])]; + tensor var_946 = cast(dtype = var_946_dtype_0, x = var_945)[name = tensor("cast_168")]; + tensor use_roll_23 = reshape(shape = var_947, x = var_946)[name = tensor("use_roll_23")]; + tensor var_949 = sub(x = var_18, y = use_roll_23)[name = tensor("op_949")]; + tensor var_950 = mul(x = filled_window_23, y = var_949)[name = tensor("op_950")]; + tensor var_951 = mul(x = rolled_23, y = use_roll_23)[name = tensor("op_951")]; + tensor window_29 = add(x = var_950, y = var_951)[name = tensor("window_29")]; + tensor var_953 = const()[name = tensor("op_953"), val = tensor(0x1p+0)]; + tensor var_954 = add(x = clip_21, y = var_953)[name = tensor("op_954")]; + tensor clip_23 = clip(alpha = var_22, beta = var_17, x = var_954)[name = tensor("clip_23")]; + tensor var_957_begin_0 = const()[name = tensor("op_957_begin_0"), val = tensor([0, 2, 0])]; + tensor var_957_end_0 = const()[name = tensor("op_957_end_0"), val = tensor([1, 3, 256])]; + tensor var_957_end_mask_0 = const()[name = tensor("op_957_end_mask_0"), val = tensor([true, false, true])]; + tensor var_957 = slice_by_index(begin = var_957_begin_0, end = var_957_end_0, end_mask = var_957_end_mask_0, x = x_15)[name = tensor("op_957")]; + tensor var_960_begin_0 = const()[name = tensor("op_960_begin_0"), val = tensor([0, 1, 0])]; + tensor var_960_end_0 = const()[name = tensor("op_960_end_0"), val = tensor([1, 16, 256])]; + tensor var_960_end_mask_0 = const()[name = tensor("op_960_end_mask_0"), val = tensor([true, true, true])]; + tensor var_960 = slice_by_index(begin = var_960_begin_0, end = var_960_end_0, end_mask = var_960_end_mask_0, x = window_29)[name = tensor("op_960")]; tensor rolled_25_interleave_0 = const()[name = tensor("rolled_25_interleave_0"), val = tensor(false)]; - tensor rolled_25 = concat(axis = var_34, interleave = rolled_25_interleave_0, values = (var_1498, var_1495))[name = tensor("rolled_25")]; + tensor rolled_25 = concat(axis = var_34, interleave = rolled_25_interleave_0, values = (var_960, var_957))[name = tensor("rolled_25")]; tensor clip_24 = clip(alpha = var_22, beta = var_21, x = clip_23)[name = tensor("clip_24")]; - tensor var_1503 = equal(x = encoder__conv_positions, y = clip_24)[name = tensor("op_1503")]; - tensor var_1504_dtype_0 = const()[name = tensor("op_1504_dtype_0"), val = tensor("fp32")]; - tensor var_1505 = const()[name = tensor("op_1505"), val = tensor([1, 16, 1])]; - tensor var_1504 = cast(dtype = var_1504_dtype_0, x = var_1503)[name = tensor("cast_167")]; - tensor fill_mask_25 = reshape(shape = var_1505, x = var_1504)[name = tensor("fill_mask_25")]; + tensor var_965 = equal(x = encoder__conv_positions, y = clip_24)[name = tensor("op_965")]; + tensor var_966_dtype_0 = const()[name = tensor("op_966_dtype_0"), val = tensor("fp32")]; + tensor var_967 = const()[name = tensor("op_967"), val = tensor([1, 16, 1])]; + tensor var_966 = cast(dtype = var_966_dtype_0, x = var_965)[name = tensor("cast_167")]; + tensor fill_mask_25 = reshape(shape = var_967, x = var_966)[name = tensor("fill_mask_25")]; tensor x_expanded_25_reps_0 = const()[name = tensor("x_expanded_25_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_25 = tile(reps = x_expanded_25_reps_0, x = var_1495)[name = tensor("x_expanded_25")]; - tensor var_1509 = sub(x = var_18, y = fill_mask_25)[name = tensor("op_1509")]; - tensor var_1510 = mul(x = input_191, y = var_1509)[name = tensor("op_1510")]; - tensor var_1511 = mul(x = x_expanded_25, y = fill_mask_25)[name = tensor("op_1511")]; - tensor filled_window_25 = add(x = var_1510, y = var_1511)[name = tensor("filled_window_25")]; - tensor var_1513 = greater_equal(x = clip_23, y = var_17)[name = tensor("op_1513")]; - tensor var_1514_dtype_0 = const()[name = tensor("op_1514_dtype_0"), val = tensor("fp32")]; - tensor var_1515 = const()[name = tensor("op_1515"), val = tensor([1, 1, 1])]; - tensor var_1514 = cast(dtype = var_1514_dtype_0, x = var_1513)[name = tensor("cast_166")]; - tensor use_roll_25 = reshape(shape = var_1515, x = var_1514)[name = tensor("use_roll_25")]; - tensor var_1517 = sub(x = var_18, y = use_roll_25)[name = tensor("op_1517")]; - tensor var_1518 = mul(x = filled_window_25, y = var_1517)[name = tensor("op_1518")]; - tensor var_1519 = mul(x = rolled_25, y = use_roll_25)[name = tensor("op_1519")]; - tensor input_201 = add(x = var_1518, y = var_1519)[name = tensor("input_201")]; - tensor var_1521 = const()[name = tensor("op_1521"), val = tensor(0x1p+0)]; - tensor var_1522 = add(x = clip_23, y = var_1521)[name = tensor("op_1522")]; - tensor clip_25 = clip(alpha = var_22, beta = var_17, x = var_1522)[name = tensor("clip_25")]; - tensor x_37_axes_0 = const()[name = tensor("x_37_axes_0"), val = tensor([-1])]; - tensor x_37 = layer_norm(axes = x_37_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_201)[name = tensor("x_37")]; - tensor input_203_perm_0 = const()[name = tensor("input_203_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_83_pad_type_0 = const()[name = tensor("inputs_83_pad_type_0"), val = tensor("valid")]; - tensor inputs_83_strides_0 = const()[name = tensor("inputs_83_strides_0"), val = tensor([1])]; - tensor inputs_83_pad_0 = const()[name = tensor("inputs_83_pad_0"), val = tensor([0, 0])]; - tensor inputs_83_dilations_0 = const()[name = tensor("inputs_83_dilations_0"), val = tensor([1])]; - tensor inputs_83_groups_0 = const()[name = tensor("inputs_83_groups_0"), val = tensor(1)]; - tensor input_203 = transpose(perm = input_203_perm_0, x = x_37)[name = tensor("transpose_50")]; - tensor inputs_83 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_83_dilations_0, groups = inputs_83_groups_0, pad = inputs_83_pad_0, pad_type = inputs_83_pad_type_0, strides = inputs_83_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_203)[name = tensor("inputs_83")]; - tensor var_1543_split_sizes_0 = const()[name = tensor("op_1543_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1543_axis_0 = const()[name = tensor("op_1543_axis_0"), val = tensor(1)]; - tensor var_1543_0, tensor var_1543_1 = split(axis = var_1543_axis_0, split_sizes = var_1543_split_sizes_0, x = inputs_83)[name = tensor("op_1543")]; - tensor var_1545 = sigmoid(x = var_1543_1)[name = tensor("op_1545")]; - tensor inputs_85 = mul(x = var_1543_0, y = var_1545)[name = tensor("inputs_85")]; - tensor outputs_aug_25_pad_type_0 = const()[name = tensor("outputs_aug_25_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_25_pad_0 = const()[name = tensor("outputs_aug_25_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_25_groups_0 = const()[name = tensor("outputs_aug_25_groups_0"), val = tensor(256)]; - tensor outputs_aug_25_strides_0 = const()[name = tensor("outputs_aug_25_strides_0"), val = tensor([1])]; - tensor outputs_aug_25_dilations_0 = const()[name = tensor("outputs_aug_25_dilations_0"), val = tensor([1])]; - tensor outputs_aug_25 = conv(dilations = outputs_aug_25_dilations_0, groups = outputs_aug_25_groups_0, pad = outputs_aug_25_pad_0, pad_type = outputs_aug_25_pad_type_0, strides = outputs_aug_25_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_85)[name = tensor("outputs_aug_25")]; - tensor input_205_begin_0 = const()[name = tensor("input_205_begin_0"), val = tensor([0, 0, 0])]; - tensor input_205_end_0 = const()[name = tensor("input_205_end_0"), val = tensor([1, 256, 16])]; - tensor input_205_end_mask_0 = const()[name = tensor("input_205_end_mask_0"), val = tensor([true, true, false])]; - tensor input_205 = slice_by_index(begin = input_205_begin_0, end = input_205_end_0, end_mask = input_205_end_mask_0, x = outputs_aug_25)[name = tensor("input_205")]; - tensor inputs_87 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_205)[name = tensor("inputs_87")]; - tensor input_207 = silu(x = inputs_87)[name = tensor("input_207")]; - tensor input_209_pad_type_0 = const()[name = tensor("input_209_pad_type_0"), val = tensor("valid")]; - tensor input_209_strides_0 = const()[name = tensor("input_209_strides_0"), val = tensor([1])]; - tensor input_209_pad_0 = const()[name = tensor("input_209_pad_0"), val = tensor([0, 0])]; - tensor input_209_dilations_0 = const()[name = tensor("input_209_dilations_0"), val = tensor([1])]; - tensor input_209_groups_0 = const()[name = tensor("input_209_groups_0"), val = tensor(1)]; - tensor input_209 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_209_dilations_0, groups = input_209_groups_0, pad = input_209_pad_0, pad_type = input_209_pad_type_0, strides = input_209_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_207)[name = tensor("input_209")]; - tensor conv_out_25_perm_0 = const()[name = tensor("conv_out_25_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1576_begin_0 = const()[name = tensor("op_1576_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1576_end_0 = const()[name = tensor("op_1576_end_0"), val = tensor([1, 16, 256])]; - tensor var_1576_end_mask_0 = const()[name = tensor("op_1576_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_25 = transpose(perm = conv_out_25_perm_0, x = input_209)[name = tensor("transpose_49")]; - tensor var_1576 = slice_by_index(begin = var_1576_begin_0, end = var_1576_end_0, end_mask = var_1576_end_mask_0, x = conv_out_25)[name = tensor("op_1576")]; - tensor var_1579_begin_0 = const()[name = tensor("op_1579_begin_0"), val = tensor([0, 3, 0])]; - tensor var_1579_end_0 = const()[name = tensor("op_1579_end_0"), val = tensor([1, 4, 256])]; - tensor var_1579_end_mask_0 = const()[name = tensor("op_1579_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1579 = slice_by_index(begin = var_1579_begin_0, end = var_1579_end_0, end_mask = var_1579_end_mask_0, x = x_31)[name = tensor("op_1579")]; - tensor var_1582_begin_0 = const()[name = tensor("op_1582_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1582_end_0 = const()[name = tensor("op_1582_end_0"), val = tensor([1, 16, 256])]; - tensor var_1582_end_mask_0 = const()[name = tensor("op_1582_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1582 = slice_by_index(begin = var_1582_begin_0, end = var_1582_end_0, end_mask = var_1582_end_mask_0, x = input_201)[name = tensor("op_1582")]; + tensor x_expanded_25 = tile(reps = x_expanded_25_reps_0, x = var_957)[name = tensor("x_expanded_25")]; + tensor var_971 = sub(x = var_18, y = fill_mask_25)[name = tensor("op_971")]; + tensor var_972 = mul(x = window_29, y = var_971)[name = tensor("op_972")]; + tensor var_973 = mul(x = x_expanded_25, y = fill_mask_25)[name = tensor("op_973")]; + tensor filled_window_25 = add(x = var_972, y = var_973)[name = tensor("filled_window_25")]; + tensor var_975 = greater_equal(x = clip_23, y = var_17)[name = tensor("op_975")]; + tensor var_976_dtype_0 = const()[name = tensor("op_976_dtype_0"), val = tensor("fp32")]; + tensor var_977 = const()[name = tensor("op_977"), val = tensor([1, 1, 1])]; + tensor var_976 = cast(dtype = var_976_dtype_0, x = var_975)[name = tensor("cast_166")]; + tensor use_roll_25 = reshape(shape = var_977, x = var_976)[name = tensor("use_roll_25")]; + tensor var_979 = sub(x = var_18, y = use_roll_25)[name = tensor("op_979")]; + tensor var_980 = mul(x = filled_window_25, y = var_979)[name = tensor("op_980")]; + tensor var_981 = mul(x = rolled_25, y = use_roll_25)[name = tensor("op_981")]; + tensor window_31 = add(x = var_980, y = var_981)[name = tensor("window_31")]; + tensor var_983 = const()[name = tensor("op_983"), val = tensor(0x1p+0)]; + tensor var_984 = add(x = clip_23, y = var_983)[name = tensor("op_984")]; + tensor clip_25 = clip(alpha = var_22, beta = var_17, x = var_984)[name = tensor("clip_25")]; + tensor var_987_begin_0 = const()[name = tensor("op_987_begin_0"), val = tensor([0, 3, 0])]; + tensor var_987_end_0 = const()[name = tensor("op_987_end_0"), val = tensor([1, 4, 256])]; + tensor var_987_end_mask_0 = const()[name = tensor("op_987_end_mask_0"), val = tensor([true, false, true])]; + tensor var_987 = slice_by_index(begin = var_987_begin_0, end = var_987_end_0, end_mask = var_987_end_mask_0, x = x_15)[name = tensor("op_987")]; + tensor var_990_begin_0 = const()[name = tensor("op_990_begin_0"), val = tensor([0, 1, 0])]; + tensor var_990_end_0 = const()[name = tensor("op_990_end_0"), val = tensor([1, 16, 256])]; + tensor var_990_end_mask_0 = const()[name = tensor("op_990_end_mask_0"), val = tensor([true, true, true])]; + tensor var_990 = slice_by_index(begin = var_990_begin_0, end = var_990_end_0, end_mask = var_990_end_mask_0, x = window_31)[name = tensor("op_990")]; tensor rolled_27_interleave_0 = const()[name = tensor("rolled_27_interleave_0"), val = tensor(false)]; - tensor rolled_27 = concat(axis = var_34, interleave = rolled_27_interleave_0, values = (var_1582, var_1579))[name = tensor("rolled_27")]; + tensor rolled_27 = concat(axis = var_34, interleave = rolled_27_interleave_0, values = (var_990, var_987))[name = tensor("rolled_27")]; tensor clip_26 = clip(alpha = var_22, beta = var_21, x = clip_25)[name = tensor("clip_26")]; - tensor var_1587 = equal(x = encoder__conv_positions, y = clip_26)[name = tensor("op_1587")]; - tensor var_1588_dtype_0 = const()[name = tensor("op_1588_dtype_0"), val = tensor("fp32")]; - tensor var_1589 = const()[name = tensor("op_1589"), val = tensor([1, 16, 1])]; - tensor var_1588 = cast(dtype = var_1588_dtype_0, x = var_1587)[name = tensor("cast_165")]; - tensor fill_mask_27 = reshape(shape = var_1589, x = var_1588)[name = tensor("fill_mask_27")]; + tensor var_995 = equal(x = encoder__conv_positions, y = clip_26)[name = tensor("op_995")]; + tensor var_996_dtype_0 = const()[name = tensor("op_996_dtype_0"), val = tensor("fp32")]; + tensor var_997 = const()[name = tensor("op_997"), val = tensor([1, 16, 1])]; + tensor var_996 = cast(dtype = var_996_dtype_0, x = var_995)[name = tensor("cast_165")]; + tensor fill_mask_27 = reshape(shape = var_997, x = var_996)[name = tensor("fill_mask_27")]; tensor x_expanded_27_reps_0 = const()[name = tensor("x_expanded_27_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_27 = tile(reps = x_expanded_27_reps_0, x = var_1579)[name = tensor("x_expanded_27")]; - tensor var_1593 = sub(x = var_18, y = fill_mask_27)[name = tensor("op_1593")]; - tensor var_1594 = mul(x = input_201, y = var_1593)[name = tensor("op_1594")]; - tensor var_1595 = mul(x = x_expanded_27, y = fill_mask_27)[name = tensor("op_1595")]; - tensor filled_window_27 = add(x = var_1594, y = var_1595)[name = tensor("filled_window_27")]; - tensor var_1597 = greater_equal(x = clip_25, y = var_17)[name = tensor("op_1597")]; - tensor var_1598_dtype_0 = const()[name = tensor("op_1598_dtype_0"), val = tensor("fp32")]; - tensor var_1599 = const()[name = tensor("op_1599"), val = tensor([1, 1, 1])]; - tensor var_1598 = cast(dtype = var_1598_dtype_0, x = var_1597)[name = tensor("cast_164")]; - tensor use_roll_27 = reshape(shape = var_1599, x = var_1598)[name = tensor("use_roll_27")]; - tensor var_1601 = sub(x = var_18, y = use_roll_27)[name = tensor("op_1601")]; - tensor var_1602 = mul(x = filled_window_27, y = var_1601)[name = tensor("op_1602")]; - tensor var_1603 = mul(x = rolled_27, y = use_roll_27)[name = tensor("op_1603")]; - tensor input_211 = add(x = var_1602, y = var_1603)[name = tensor("input_211")]; - tensor var_1605 = const()[name = tensor("op_1605"), val = tensor(0x1p+0)]; - tensor var_1606 = add(x = clip_25, y = var_1605)[name = tensor("op_1606")]; - tensor clip_27 = clip(alpha = var_22, beta = var_17, x = var_1606)[name = tensor("clip_27")]; - tensor x_39_axes_0 = const()[name = tensor("x_39_axes_0"), val = tensor([-1])]; - tensor x_39 = layer_norm(axes = x_39_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_211)[name = tensor("x_39")]; - tensor input_213_perm_0 = const()[name = tensor("input_213_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_89_pad_type_0 = const()[name = tensor("inputs_89_pad_type_0"), val = tensor("valid")]; - tensor inputs_89_strides_0 = const()[name = tensor("inputs_89_strides_0"), val = tensor([1])]; - tensor inputs_89_pad_0 = const()[name = tensor("inputs_89_pad_0"), val = tensor([0, 0])]; - tensor inputs_89_dilations_0 = const()[name = tensor("inputs_89_dilations_0"), val = tensor([1])]; - tensor inputs_89_groups_0 = const()[name = tensor("inputs_89_groups_0"), val = tensor(1)]; - tensor input_213 = transpose(perm = input_213_perm_0, x = x_39)[name = tensor("transpose_48")]; - tensor inputs_89 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_89_dilations_0, groups = inputs_89_groups_0, pad = inputs_89_pad_0, pad_type = inputs_89_pad_type_0, strides = inputs_89_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_213)[name = tensor("inputs_89")]; - tensor var_1627_split_sizes_0 = const()[name = tensor("op_1627_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1627_axis_0 = const()[name = tensor("op_1627_axis_0"), val = tensor(1)]; - tensor var_1627_0, tensor var_1627_1 = split(axis = var_1627_axis_0, split_sizes = var_1627_split_sizes_0, x = inputs_89)[name = tensor("op_1627")]; - tensor var_1629 = sigmoid(x = var_1627_1)[name = tensor("op_1629")]; - tensor inputs_91 = mul(x = var_1627_0, y = var_1629)[name = tensor("inputs_91")]; - tensor outputs_aug_27_pad_type_0 = const()[name = tensor("outputs_aug_27_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_27_pad_0 = const()[name = tensor("outputs_aug_27_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_27_groups_0 = const()[name = tensor("outputs_aug_27_groups_0"), val = tensor(256)]; - tensor outputs_aug_27_strides_0 = const()[name = tensor("outputs_aug_27_strides_0"), val = tensor([1])]; - tensor outputs_aug_27_dilations_0 = const()[name = tensor("outputs_aug_27_dilations_0"), val = tensor([1])]; - tensor outputs_aug_27 = conv(dilations = outputs_aug_27_dilations_0, groups = outputs_aug_27_groups_0, pad = outputs_aug_27_pad_0, pad_type = outputs_aug_27_pad_type_0, strides = outputs_aug_27_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_91)[name = tensor("outputs_aug_27")]; - tensor input_215_begin_0 = const()[name = tensor("input_215_begin_0"), val = tensor([0, 0, 0])]; - tensor input_215_end_0 = const()[name = tensor("input_215_end_0"), val = tensor([1, 256, 16])]; - tensor input_215_end_mask_0 = const()[name = tensor("input_215_end_mask_0"), val = tensor([true, true, false])]; - tensor input_215 = slice_by_index(begin = input_215_begin_0, end = input_215_end_0, end_mask = input_215_end_mask_0, x = outputs_aug_27)[name = tensor("input_215")]; - tensor inputs_93 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_215)[name = tensor("inputs_93")]; - tensor input_217 = silu(x = inputs_93)[name = tensor("input_217")]; - tensor input_219_pad_type_0 = const()[name = tensor("input_219_pad_type_0"), val = tensor("valid")]; - tensor input_219_strides_0 = const()[name = tensor("input_219_strides_0"), val = tensor([1])]; - tensor input_219_pad_0 = const()[name = tensor("input_219_pad_0"), val = tensor([0, 0])]; - tensor input_219_dilations_0 = const()[name = tensor("input_219_dilations_0"), val = tensor([1])]; - tensor input_219_groups_0 = const()[name = tensor("input_219_groups_0"), val = tensor(1)]; - tensor input_219 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_219_dilations_0, groups = input_219_groups_0, pad = input_219_pad_0, pad_type = input_219_pad_type_0, strides = input_219_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_217)[name = tensor("input_219")]; - tensor conv_out_27_perm_0 = const()[name = tensor("conv_out_27_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1660_begin_0 = const()[name = tensor("op_1660_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1660_end_0 = const()[name = tensor("op_1660_end_0"), val = tensor([1, 16, 256])]; - tensor var_1660_end_mask_0 = const()[name = tensor("op_1660_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_27 = transpose(perm = conv_out_27_perm_0, x = input_219)[name = tensor("transpose_47")]; - tensor var_1660 = slice_by_index(begin = var_1660_begin_0, end = var_1660_end_0, end_mask = var_1660_end_mask_0, x = conv_out_27)[name = tensor("op_1660")]; - tensor var_1663_begin_0 = const()[name = tensor("op_1663_begin_0"), val = tensor([0, 4, 0])]; - tensor var_1663_end_0 = const()[name = tensor("op_1663_end_0"), val = tensor([1, 1, 256])]; - tensor var_1663_end_mask_0 = const()[name = tensor("op_1663_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1663 = slice_by_index(begin = var_1663_begin_0, end = var_1663_end_0, end_mask = var_1663_end_mask_0, x = x_31)[name = tensor("op_1663")]; - tensor var_1666_begin_0 = const()[name = tensor("op_1666_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1666_end_0 = const()[name = tensor("op_1666_end_0"), val = tensor([1, 16, 256])]; - tensor var_1666_end_mask_0 = const()[name = tensor("op_1666_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1666 = slice_by_index(begin = var_1666_begin_0, end = var_1666_end_0, end_mask = var_1666_end_mask_0, x = input_211)[name = tensor("op_1666")]; + tensor x_expanded_27 = tile(reps = x_expanded_27_reps_0, x = var_987)[name = tensor("x_expanded_27")]; + tensor var_1001 = sub(x = var_18, y = fill_mask_27)[name = tensor("op_1001")]; + tensor var_1002 = mul(x = window_31, y = var_1001)[name = tensor("op_1002")]; + tensor var_1003 = mul(x = x_expanded_27, y = fill_mask_27)[name = tensor("op_1003")]; + tensor filled_window_27 = add(x = var_1002, y = var_1003)[name = tensor("filled_window_27")]; + tensor var_1005 = greater_equal(x = clip_25, y = var_17)[name = tensor("op_1005")]; + tensor var_1006_dtype_0 = const()[name = tensor("op_1006_dtype_0"), val = tensor("fp32")]; + tensor var_1007 = const()[name = tensor("op_1007"), val = tensor([1, 1, 1])]; + tensor var_1006 = cast(dtype = var_1006_dtype_0, x = var_1005)[name = tensor("cast_164")]; + tensor use_roll_27 = reshape(shape = var_1007, x = var_1006)[name = tensor("use_roll_27")]; + tensor var_1009 = sub(x = var_18, y = use_roll_27)[name = tensor("op_1009")]; + tensor var_1010 = mul(x = filled_window_27, y = var_1009)[name = tensor("op_1010")]; + tensor var_1011 = mul(x = rolled_27, y = use_roll_27)[name = tensor("op_1011")]; + tensor window_33 = add(x = var_1010, y = var_1011)[name = tensor("window_33")]; + tensor var_1013 = const()[name = tensor("op_1013"), val = tensor(0x1p+0)]; + tensor var_1014 = add(x = clip_25, y = var_1013)[name = tensor("op_1014")]; + tensor clip_27 = clip(alpha = var_22, beta = var_17, x = var_1014)[name = tensor("clip_27")]; + tensor var_1017_begin_0 = const()[name = tensor("op_1017_begin_0"), val = tensor([0, 4, 0])]; + tensor var_1017_end_0 = const()[name = tensor("op_1017_end_0"), val = tensor([1, 1, 256])]; + tensor var_1017_end_mask_0 = const()[name = tensor("op_1017_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1017 = slice_by_index(begin = var_1017_begin_0, end = var_1017_end_0, end_mask = var_1017_end_mask_0, x = x_15)[name = tensor("op_1017")]; + tensor var_1020_begin_0 = const()[name = tensor("op_1020_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1020_end_0 = const()[name = tensor("op_1020_end_0"), val = tensor([1, 16, 256])]; + tensor var_1020_end_mask_0 = const()[name = tensor("op_1020_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1020 = slice_by_index(begin = var_1020_begin_0, end = var_1020_end_0, end_mask = var_1020_end_mask_0, x = window_33)[name = tensor("op_1020")]; tensor rolled_29_interleave_0 = const()[name = tensor("rolled_29_interleave_0"), val = tensor(false)]; - tensor rolled_29 = concat(axis = var_34, interleave = rolled_29_interleave_0, values = (var_1666, var_1663))[name = tensor("rolled_29")]; + tensor rolled_29 = concat(axis = var_34, interleave = rolled_29_interleave_0, values = (var_1020, var_1017))[name = tensor("rolled_29")]; tensor clip_28 = clip(alpha = var_22, beta = var_21, x = clip_27)[name = tensor("clip_28")]; - tensor var_1671 = equal(x = encoder__conv_positions, y = clip_28)[name = tensor("op_1671")]; - tensor var_1672_dtype_0 = const()[name = tensor("op_1672_dtype_0"), val = tensor("fp32")]; - tensor var_1673 = const()[name = tensor("op_1673"), val = tensor([1, 16, 1])]; - tensor var_1672 = cast(dtype = var_1672_dtype_0, x = var_1671)[name = tensor("cast_163")]; - tensor fill_mask_29 = reshape(shape = var_1673, x = var_1672)[name = tensor("fill_mask_29")]; + tensor var_1025 = equal(x = encoder__conv_positions, y = clip_28)[name = tensor("op_1025")]; + tensor var_1026_dtype_0 = const()[name = tensor("op_1026_dtype_0"), val = tensor("fp32")]; + tensor var_1027 = const()[name = tensor("op_1027"), val = tensor([1, 16, 1])]; + tensor var_1026 = cast(dtype = var_1026_dtype_0, x = var_1025)[name = tensor("cast_163")]; + tensor fill_mask_29 = reshape(shape = var_1027, x = var_1026)[name = tensor("fill_mask_29")]; tensor x_expanded_29_reps_0 = const()[name = tensor("x_expanded_29_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_29 = tile(reps = x_expanded_29_reps_0, x = var_1663)[name = tensor("x_expanded_29")]; - tensor var_1677 = sub(x = var_18, y = fill_mask_29)[name = tensor("op_1677")]; - tensor var_1678 = mul(x = input_211, y = var_1677)[name = tensor("op_1678")]; - tensor var_1679 = mul(x = x_expanded_29, y = fill_mask_29)[name = tensor("op_1679")]; - tensor filled_window_29 = add(x = var_1678, y = var_1679)[name = tensor("filled_window_29")]; - tensor var_1681 = greater_equal(x = clip_27, y = var_17)[name = tensor("op_1681")]; - tensor var_1682_dtype_0 = const()[name = tensor("op_1682_dtype_0"), val = tensor("fp32")]; - tensor var_1683 = const()[name = tensor("op_1683"), val = tensor([1, 1, 1])]; - tensor var_1682 = cast(dtype = var_1682_dtype_0, x = var_1681)[name = tensor("cast_162")]; - tensor use_roll_29 = reshape(shape = var_1683, x = var_1682)[name = tensor("use_roll_29")]; - tensor var_1685 = sub(x = var_18, y = use_roll_29)[name = tensor("op_1685")]; - tensor var_1686 = mul(x = filled_window_29, y = var_1685)[name = tensor("op_1686")]; - tensor var_1687 = mul(x = rolled_29, y = use_roll_29)[name = tensor("op_1687")]; - tensor input_221 = add(x = var_1686, y = var_1687)[name = tensor("input_221")]; - tensor var_1689 = const()[name = tensor("op_1689"), val = tensor(0x1p+0)]; - tensor var_1690 = add(x = clip_27, y = var_1689)[name = tensor("op_1690")]; - tensor clip_29 = clip(alpha = var_22, beta = var_17, x = var_1690)[name = tensor("clip_29")]; - tensor x_41_axes_0 = const()[name = tensor("x_41_axes_0"), val = tensor([-1])]; - tensor x_41 = layer_norm(axes = x_41_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_221)[name = tensor("x_41")]; - tensor input_223_perm_0 = const()[name = tensor("input_223_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_95_pad_type_0 = const()[name = tensor("inputs_95_pad_type_0"), val = tensor("valid")]; - tensor inputs_95_strides_0 = const()[name = tensor("inputs_95_strides_0"), val = tensor([1])]; - tensor inputs_95_pad_0 = const()[name = tensor("inputs_95_pad_0"), val = tensor([0, 0])]; - tensor inputs_95_dilations_0 = const()[name = tensor("inputs_95_dilations_0"), val = tensor([1])]; - tensor inputs_95_groups_0 = const()[name = tensor("inputs_95_groups_0"), val = tensor(1)]; - tensor input_223 = transpose(perm = input_223_perm_0, x = x_41)[name = tensor("transpose_46")]; - tensor inputs_95 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_95_dilations_0, groups = inputs_95_groups_0, pad = inputs_95_pad_0, pad_type = inputs_95_pad_type_0, strides = inputs_95_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_223)[name = tensor("inputs_95")]; - tensor var_1711_split_sizes_0 = const()[name = tensor("op_1711_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1711_axis_0 = const()[name = tensor("op_1711_axis_0"), val = tensor(1)]; - tensor var_1711_0, tensor var_1711_1 = split(axis = var_1711_axis_0, split_sizes = var_1711_split_sizes_0, x = inputs_95)[name = tensor("op_1711")]; - tensor var_1713 = sigmoid(x = var_1711_1)[name = tensor("op_1713")]; - tensor inputs_97 = mul(x = var_1711_0, y = var_1713)[name = tensor("inputs_97")]; - tensor outputs_aug_29_pad_type_0 = const()[name = tensor("outputs_aug_29_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_29_pad_0 = const()[name = tensor("outputs_aug_29_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_29_groups_0 = const()[name = tensor("outputs_aug_29_groups_0"), val = tensor(256)]; - tensor outputs_aug_29_strides_0 = const()[name = tensor("outputs_aug_29_strides_0"), val = tensor([1])]; - tensor outputs_aug_29_dilations_0 = const()[name = tensor("outputs_aug_29_dilations_0"), val = tensor([1])]; - tensor outputs_aug_29 = conv(dilations = outputs_aug_29_dilations_0, groups = outputs_aug_29_groups_0, pad = outputs_aug_29_pad_0, pad_type = outputs_aug_29_pad_type_0, strides = outputs_aug_29_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_97)[name = tensor("outputs_aug_29")]; - tensor input_225_begin_0 = const()[name = tensor("input_225_begin_0"), val = tensor([0, 0, 0])]; - tensor input_225_end_0 = const()[name = tensor("input_225_end_0"), val = tensor([1, 256, 16])]; - tensor input_225_end_mask_0 = const()[name = tensor("input_225_end_mask_0"), val = tensor([true, true, false])]; - tensor input_225 = slice_by_index(begin = input_225_begin_0, end = input_225_end_0, end_mask = input_225_end_mask_0, x = outputs_aug_29)[name = tensor("input_225")]; - tensor inputs_99 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_225)[name = tensor("inputs_99")]; - tensor input_227 = silu(x = inputs_99)[name = tensor("input_227")]; - tensor input_229_pad_type_0 = const()[name = tensor("input_229_pad_type_0"), val = tensor("valid")]; - tensor input_229_strides_0 = const()[name = tensor("input_229_strides_0"), val = tensor([1])]; - tensor input_229_pad_0 = const()[name = tensor("input_229_pad_0"), val = tensor([0, 0])]; - tensor input_229_dilations_0 = const()[name = tensor("input_229_dilations_0"), val = tensor([1])]; - tensor input_229_groups_0 = const()[name = tensor("input_229_groups_0"), val = tensor(1)]; - tensor input_229 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_229_dilations_0, groups = input_229_groups_0, pad = input_229_pad_0, pad_type = input_229_pad_type_0, strides = input_229_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_227)[name = tensor("input_229")]; - tensor conv_out_29_perm_0 = const()[name = tensor("conv_out_29_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1744_begin_0 = const()[name = tensor("op_1744_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1744_end_0 = const()[name = tensor("op_1744_end_0"), val = tensor([1, 16, 256])]; - tensor var_1744_end_mask_0 = const()[name = tensor("op_1744_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_29 = transpose(perm = conv_out_29_perm_0, x = input_229)[name = tensor("transpose_45")]; - tensor var_1744 = slice_by_index(begin = var_1744_begin_0, end = var_1744_end_0, end_mask = var_1744_end_mask_0, x = conv_out_29)[name = tensor("op_1744")]; - tensor var_1747_interleave_0 = const()[name = tensor("op_1747_interleave_0"), val = tensor(false)]; - tensor var_1747 = concat(axis = var_34, interleave = var_1747_interleave_0, values = (var_1408, var_1492, var_1576, var_1660, var_1744))[name = tensor("op_1747")]; - tensor input_231 = add(x = x_31, y = var_1747)[name = tensor("input_231")]; - tensor input_233_axes_0 = const()[name = tensor("input_233_axes_0"), val = tensor([-1])]; - tensor input_233 = layer_norm(axes = input_233_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_231)[name = tensor("input_233")]; - tensor inputs_101 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_233)[name = tensor("linear_26")]; - tensor input_235 = silu(x = inputs_101)[name = tensor("input_235")]; - tensor input_239 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_235)[name = tensor("linear_27")]; - tensor var_1770 = const()[name = tensor("op_1770"), val = tensor(0x1p-1)]; - tensor var_1771 = mul(x = input_239, y = var_1770)[name = tensor("op_1771")]; - tensor input_241 = add(x = var_1771, y = input_231)[name = tensor("input_241")]; - tensor input_243_axes_0 = const()[name = tensor("input_243_axes_0"), val = tensor([-1])]; - tensor input_243 = layer_norm(axes = input_243_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_36, gamma = encoder_layer_norm_2_weight, x = input_241)[name = tensor("input_243")]; - tensor input_245_axes_0 = const()[name = tensor("input_245_axes_0"), val = tensor([-1])]; - tensor input_245 = layer_norm(axes = input_245_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_243)[name = tensor("input_245")]; - tensor inputs_103 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_245)[name = tensor("linear_28")]; - tensor input_247 = silu(x = inputs_103)[name = tensor("input_247")]; - tensor input_251 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_247)[name = tensor("linear_29")]; - tensor var_1800 = const()[name = tensor("op_1800"), val = tensor(0x1p-1)]; - tensor var_1801 = mul(x = input_251, y = var_1800)[name = tensor("op_1801")]; - tensor input_253 = add(x = var_1801, y = input_243)[name = tensor("input_253")]; - tensor x_43_axes_0 = const()[name = tensor("x_43_axes_0"), val = tensor([-1])]; - tensor x_43 = layer_norm(axes = x_43_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_36, gamma = encoder_ret_lns_3_weight, x = input_253)[name = tensor("x_43")]; + tensor x_expanded_29 = tile(reps = x_expanded_29_reps_0, x = var_1017)[name = tensor("x_expanded_29")]; + tensor var_1031 = sub(x = var_18, y = fill_mask_29)[name = tensor("op_1031")]; + tensor var_1032 = mul(x = window_33, y = var_1031)[name = tensor("op_1032")]; + tensor var_1033 = mul(x = x_expanded_29, y = fill_mask_29)[name = tensor("op_1033")]; + tensor filled_window_29 = add(x = var_1032, y = var_1033)[name = tensor("filled_window_29")]; + tensor var_1035 = greater_equal(x = clip_27, y = var_17)[name = tensor("op_1035")]; + tensor var_1036_dtype_0 = const()[name = tensor("op_1036_dtype_0"), val = tensor("fp32")]; + tensor var_1037 = const()[name = tensor("op_1037"), val = tensor([1, 1, 1])]; + tensor var_1036 = cast(dtype = var_1036_dtype_0, x = var_1035)[name = tensor("cast_162")]; + tensor use_roll_29 = reshape(shape = var_1037, x = var_1036)[name = tensor("use_roll_29")]; + tensor var_1039 = sub(x = var_18, y = use_roll_29)[name = tensor("op_1039")]; + tensor var_1040 = mul(x = filled_window_29, y = var_1039)[name = tensor("op_1040")]; + tensor var_1041 = mul(x = rolled_29, y = use_roll_29)[name = tensor("op_1041")]; + tensor window_35 = add(x = var_1040, y = var_1041)[name = tensor("window_35")]; + tensor var_1043 = const()[name = tensor("op_1043"), val = tensor(0x1p+0)]; + tensor var_1044 = add(x = clip_27, y = var_1043)[name = tensor("op_1044")]; + tensor clip_29 = clip(alpha = var_22, beta = var_17, x = var_1044)[name = tensor("clip_29")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_31, interleave = input_101_interleave_0, values = (window_27, window_29, window_31, window_33, window_35))[name = tensor("input_101")]; + tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; + tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; + tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_23_pad_type_0 = const()[name = tensor("inputs_23_pad_type_0"), val = tensor("valid")]; + tensor inputs_23_strides_0 = const()[name = tensor("inputs_23_strides_0"), val = tensor([1])]; + tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; + tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; + tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; + tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; + tensor var_1067_split_sizes_0 = const()[name = tensor("op_1067_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1067_axis_0 = const()[name = tensor("op_1067_axis_0"), val = tensor(1)]; + tensor var_1067_0, tensor var_1067_1 = split(axis = var_1067_axis_0, split_sizes = var_1067_split_sizes_0, x = inputs_23)[name = tensor("op_1067")]; + tensor var_1069 = sigmoid(x = var_1067_1)[name = tensor("op_1069")]; + tensor inputs_25 = mul(x = var_1067_0, y = var_1069)[name = tensor("inputs_25")]; + tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; + tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; + tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; + tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_25)[name = tensor("outputs_aug_5")]; + tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; + tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([5, 256, 16])]; + tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; + tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_5)[name = tensor("input_105")]; + tensor inputs_27 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_105)[name = tensor("inputs_27")]; + tensor input_107 = silu(x = inputs_27)[name = tensor("input_107")]; + tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; + tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; + tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; + tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; + tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; + tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; + tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1100_begin_0 = const()[name = tensor("op_1100_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1100_end_0 = const()[name = tensor("op_1100_end_0"), val = tensor([5, 16, 256])]; + tensor var_1100_end_mask_0 = const()[name = tensor("op_1100_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_1100 = slice_by_index(begin = var_1100_begin_0, end = var_1100_end_0, end_mask = var_1100_end_mask_0, x = conv_out_5)[name = tensor("op_1100")]; + tensor var_1102_perm_0 = const()[name = tensor("op_1102_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1102 = transpose(perm = var_1102_perm_0, x = var_1100)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_1102)[name = tensor("input_111")]; + tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; + tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; + tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; + tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; + tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; + tensor var_1125 = const()[name = tensor("op_1125"), val = tensor(0x1p-1)]; + tensor var_1126 = mul(x = input_119, y = var_1125)[name = tensor("op_1126")]; + tensor input_121 = add(x = var_1126, y = input_111)[name = tensor("input_121")]; + tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; + tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_36, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; + tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; + tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; + tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; + tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; + tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; + tensor var_1155 = const()[name = tensor("op_1155"), val = tensor(0x1p-1)]; + tensor var_1156 = mul(x = input_131, y = var_1155)[name = tensor("op_1156")]; + tensor input_133 = add(x = var_1156, y = input_123)[name = tensor("input_133")]; + tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; + tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_36, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; tensor prev_kv_7_end_0 = const()[name = tensor("prev_kv_7_end_0"), val = tensor([4, 1, 4, 64, 64])]; tensor prev_kv_7_end_mask_0 = const()[name = tensor("prev_kv_7_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1641,455 +1180,301 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_1815 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_43)[name = tensor("linear_30")]; - tensor var_1816 = const()[name = tensor("op_1816"), val = tensor([1, 5, 4, 64])]; - tensor var_1817 = reshape(shape = var_1816, x = var_1815)[name = tensor("op_1817")]; + tensor var_1170 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_1171 = const()[name = tensor("op_1171"), val = tensor([1, 5, 4, 64])]; + tensor var_1172 = reshape(shape = var_1171, x = var_1170)[name = tensor("op_1172")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1821 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_43)[name = tensor("linear_31")]; - tensor var_1822 = const()[name = tensor("op_1822"), val = tensor(0x1p-3)]; - tensor var_1823 = mul(x = var_1821, y = var_1822)[name = tensor("op_1823")]; - tensor var_1824 = const()[name = tensor("op_1824"), val = tensor([1, 5, 4, 64])]; - tensor var_1825 = reshape(shape = var_1824, x = var_1823)[name = tensor("op_1825")]; + tensor var_1176 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_1177 = const()[name = tensor("op_1177"), val = tensor(0x1p-3)]; + tensor var_1178 = mul(x = var_1176, y = var_1177)[name = tensor("op_1178")]; + tensor var_1179 = const()[name = tensor("op_1179"), val = tensor([1, 5, 4, 64])]; + tensor var_1180 = reshape(shape = var_1179, x = var_1178)[name = tensor("op_1180")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1829 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_43)[name = tensor("linear_32")]; - tensor var_1830 = const()[name = tensor("op_1830"), val = tensor([1, 5, 4, 64])]; - tensor var_1831 = reshape(shape = var_1830, x = var_1829)[name = tensor("op_1831")]; + tensor var_1184 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_1185 = const()[name = tensor("op_1185"), val = tensor([1, 5, 4, 64])]; + tensor var_1186 = reshape(shape = var_1185, x = var_1184)[name = tensor("op_1186")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_257 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_43)[name = tensor("linear_33")]; + tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; tensor s_t_7 = add(x = prev_scale_7, y = encoder__t_index)[name = tensor("s_t_7")]; tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_1825)[name = tensor("transpose_43")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_1817)[name = tensor("transpose_44")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_1180)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_1172)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_1841 = const()[name = tensor("op_1841"), val = tensor([5, 1])]; - tensor var_1842 = reshape(shape = var_1841, x = sqrt_s_t_7)[name = tensor("op_1842")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_1842)[name = tensor("M_7")]; - tensor var_1844 = mul(x = qk_7, y = M_7)[name = tensor("op_1844")]; + tensor var_1196 = const()[name = tensor("op_1196"), val = tensor([5, 1])]; + tensor var_1197 = reshape(shape = var_1196, x = sqrt_s_t_7)[name = tensor("op_1197")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_1197)[name = tensor("M_7")]; + tensor var_1199 = mul(x = qk_7, y = M_7)[name = tensor("op_1199")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_1831)[name = tensor("transpose_42")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1844, y = v_7)[name = tensor("inner_7")]; - tensor var_1846_transpose_x_0 = const()[name = tensor("op_1846_transpose_x_0"), val = tensor(false)]; - tensor var_1846_transpose_y_0 = const()[name = tensor("op_1846_transpose_y_0"), val = tensor(false)]; - tensor var_1846 = matmul(transpose_x = var_1846_transpose_x_0, transpose_y = var_1846_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1846")]; - tensor var_1847 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1847")]; - tensor var_1848 = const()[name = tensor("op_1848"), val = tensor([1, 1, 5, 1])]; - tensor var_1849 = reshape(shape = var_1848, x = var_1847)[name = tensor("op_1849")]; - tensor cross_7 = mul(x = var_1846, y = var_1849)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_1186)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1199, y = v_7)[name = tensor("inner_7")]; + tensor var_1201_transpose_x_0 = const()[name = tensor("op_1201_transpose_x_0"), val = tensor(false)]; + tensor var_1201_transpose_y_0 = const()[name = tensor("op_1201_transpose_y_0"), val = tensor(false)]; + tensor var_1201 = matmul(transpose_x = var_1201_transpose_x_0, transpose_y = var_1201_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1201")]; + tensor var_1202 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1202")]; + tensor var_1203 = const()[name = tensor("op_1203"), val = tensor([1, 1, 5, 1])]; + tensor var_1204 = reshape(shape = var_1203, x = var_1202)[name = tensor("op_1204")]; + tensor cross_7 = mul(x = var_1201, y = var_1204)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_1852 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1852")]; - tensor var_1854_transpose_x_1 = const()[name = tensor("op_1854_transpose_x_1"), val = tensor(true)]; - tensor var_1854_transpose_y_1 = const()[name = tensor("op_1854_transpose_y_1"), val = tensor(false)]; - tensor var_1854 = matmul(transpose_x = var_1854_transpose_x_1, transpose_y = var_1854_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1854")]; - tensor new_kv_unnorm_7 = add(x = var_1852, y = var_1854)[name = tensor("new_kv_unnorm_7")]; - tensor var_1856 = const()[name = tensor("op_1856"), val = tensor(0x1.4p+2)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_1856)[name = tensor("new_scale_7")]; - tensor var_1858 = sqrt(x = new_scale_7)[name = tensor("op_1858")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1858)[name = tensor("nkv_1")]; - tensor var_1860_perm_0 = const()[name = tensor("op_1860_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1207 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1207")]; + tensor var_1209_transpose_x_1 = const()[name = tensor("op_1209_transpose_x_1"), val = tensor(true)]; + tensor var_1209_transpose_y_1 = const()[name = tensor("op_1209_transpose_y_1"), val = tensor(false)]; + tensor var_1209 = matmul(transpose_x = var_1209_transpose_x_1, transpose_y = var_1209_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1209")]; + tensor new_kv_unnorm_7 = add(x = var_1207, y = var_1209)[name = tensor("new_kv_unnorm_7")]; + tensor var_1211 = const()[name = tensor("op_1211"), val = tensor(0x1.4p+2)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_1211)[name = tensor("new_scale_7")]; + tensor var_1213 = sqrt(x = new_scale_7)[name = tensor("op_1213")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1213)[name = tensor("nkv_1")]; + tensor var_1215_perm_0 = const()[name = tensor("op_1215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_1860 = transpose(perm = var_1860_perm_0, x = out_19)[name = tensor("transpose_41")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_25, x = var_1860)[name = tensor("out_21")]; - tensor var_1864 = const()[name = tensor("op_1864"), val = tensor([1, 5, 256])]; - tensor out_23 = reshape(shape = var_1864, x = out_21)[name = tensor("out_23")]; - tensor var_1866 = silu(x = input_257)[name = tensor("op_1866")]; - tensor input_259 = mul(x = var_1866, y = out_23)[name = tensor("input_259")]; - tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_259)[name = tensor("linear_34")]; - tensor x_45 = add(x = input_253, y = ret_out_7)[name = tensor("x_45")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor var_1215 = transpose(perm = var_1215_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_25, x = var_1215)[name = tensor("out_21")]; + tensor var_1219 = const()[name = tensor("op_1219"), val = tensor([1, 5, 256])]; + tensor out_23 = reshape(shape = var_1219, x = out_21)[name = tensor("out_23")]; + tensor var_1221 = silu(x = input_137)[name = tensor("op_1221")]; + tensor input_139 = mul(x = var_1221, y = out_23)[name = tensor("input_139")]; + tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; + tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; + tensor window_37_begin_0 = const()[name = tensor("window_37_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_37_end_0 = const()[name = tensor("window_37_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_37_end_mask_0 = const()[name = tensor("window_37_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_37_squeeze_mask_0 = const()[name = tensor("window_37_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_37 = slice_by_index(begin = window_37_begin_0, end = window_37_end_0, end_mask = window_37_end_mask_0, squeeze_mask = window_37_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_37")]; tensor filled_31_begin_0 = const()[name = tensor("filled_31_begin_0"), val = tensor([3, 0])]; tensor filled_31_end_0 = const()[name = tensor("filled_31_end_0"), val = tensor([4, 1])]; tensor filled_31_end_mask_0 = const()[name = tensor("filled_31_end_mask_0"), val = tensor([false, true])]; tensor filled_31_squeeze_mask_0 = const()[name = tensor("filled_31_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_31 = slice_by_index(begin = filled_31_begin_0, end = filled_31_end_0, end_mask = filled_31_end_mask_0, squeeze_mask = filled_31_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_31")]; - tensor var_1875_begin_0 = const()[name = tensor("op_1875_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1875_end_0 = const()[name = tensor("op_1875_end_0"), val = tensor([1, 1, 256])]; - tensor var_1875_end_mask_0 = const()[name = tensor("op_1875_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1875 = slice_by_index(begin = var_1875_begin_0, end = var_1875_end_0, end_mask = var_1875_end_mask_0, x = x_45)[name = tensor("op_1875")]; - tensor var_1878_begin_0 = const()[name = tensor("op_1878_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1878_end_0 = const()[name = tensor("op_1878_end_0"), val = tensor([1, 16, 256])]; - tensor var_1878_end_mask_0 = const()[name = tensor("op_1878_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1878 = slice_by_index(begin = var_1878_begin_0, end = var_1878_end_0, end_mask = var_1878_end_mask_0, x = window)[name = tensor("op_1878")]; + tensor var_1230_begin_0 = const()[name = tensor("op_1230_begin_0"), val = tensor([0, 0, 0])]; + tensor var_1230_end_0 = const()[name = tensor("op_1230_end_0"), val = tensor([1, 1, 256])]; + tensor var_1230_end_mask_0 = const()[name = tensor("op_1230_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1230 = slice_by_index(begin = var_1230_begin_0, end = var_1230_end_0, end_mask = var_1230_end_mask_0, x = x_21)[name = tensor("op_1230")]; + tensor var_1233_begin_0 = const()[name = tensor("op_1233_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1233_end_0 = const()[name = tensor("op_1233_end_0"), val = tensor([1, 16, 256])]; + tensor var_1233_end_mask_0 = const()[name = tensor("op_1233_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1233 = slice_by_index(begin = var_1233_begin_0, end = var_1233_end_0, end_mask = var_1233_end_mask_0, x = window_37)[name = tensor("op_1233")]; tensor rolled_31_interleave_0 = const()[name = tensor("rolled_31_interleave_0"), val = tensor(false)]; - tensor rolled_31 = concat(axis = var_34, interleave = rolled_31_interleave_0, values = (var_1878, var_1875))[name = tensor("rolled_31")]; + tensor rolled_31 = concat(axis = var_34, interleave = rolled_31_interleave_0, values = (var_1233, var_1230))[name = tensor("rolled_31")]; tensor clip_30 = clip(alpha = var_22, beta = var_21, x = filled_31)[name = tensor("clip_30")]; - tensor var_1883 = equal(x = encoder__conv_positions, y = clip_30)[name = tensor("op_1883")]; - tensor var_1884_dtype_0 = const()[name = tensor("op_1884_dtype_0"), val = tensor("fp32")]; - tensor var_1885 = const()[name = tensor("op_1885"), val = tensor([1, 16, 1])]; - tensor var_1884 = cast(dtype = var_1884_dtype_0, x = var_1883)[name = tensor("cast_161")]; - tensor fill_mask_31 = reshape(shape = var_1885, x = var_1884)[name = tensor("fill_mask_31")]; + tensor var_1238 = equal(x = encoder__conv_positions, y = clip_30)[name = tensor("op_1238")]; + tensor var_1239_dtype_0 = const()[name = tensor("op_1239_dtype_0"), val = tensor("fp32")]; + tensor var_1240 = const()[name = tensor("op_1240"), val = tensor([1, 16, 1])]; + tensor var_1239 = cast(dtype = var_1239_dtype_0, x = var_1238)[name = tensor("cast_161")]; + tensor fill_mask_31 = reshape(shape = var_1240, x = var_1239)[name = tensor("fill_mask_31")]; tensor x_expanded_31_reps_0 = const()[name = tensor("x_expanded_31_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_31 = tile(reps = x_expanded_31_reps_0, x = var_1875)[name = tensor("x_expanded_31")]; - tensor var_1889 = sub(x = var_18, y = fill_mask_31)[name = tensor("op_1889")]; - tensor var_1890 = mul(x = window, y = var_1889)[name = tensor("op_1890")]; - tensor var_1891 = mul(x = x_expanded_31, y = fill_mask_31)[name = tensor("op_1891")]; - tensor filled_window_31 = add(x = var_1890, y = var_1891)[name = tensor("filled_window_31")]; - tensor var_1893 = greater_equal(x = filled_31, y = var_17)[name = tensor("op_1893")]; - tensor var_1894_dtype_0 = const()[name = tensor("op_1894_dtype_0"), val = tensor("fp32")]; - tensor var_1895 = const()[name = tensor("op_1895"), val = tensor([1, 1, 1])]; - tensor var_1894 = cast(dtype = var_1894_dtype_0, x = var_1893)[name = tensor("cast_160")]; - tensor use_roll_31 = reshape(shape = var_1895, x = var_1894)[name = tensor("use_roll_31")]; - tensor var_1897 = sub(x = var_18, y = use_roll_31)[name = tensor("op_1897")]; - tensor var_1898 = mul(x = filled_window_31, y = var_1897)[name = tensor("op_1898")]; - tensor var_1899 = mul(x = rolled_31, y = use_roll_31)[name = tensor("op_1899")]; - tensor input_261 = add(x = var_1898, y = var_1899)[name = tensor("input_261")]; - tensor var_1901 = const()[name = tensor("op_1901"), val = tensor(0x1p+0)]; - tensor var_1902 = add(x = filled_31, y = var_1901)[name = tensor("op_1902")]; - tensor clip_31 = clip(alpha = var_22, beta = var_17, x = var_1902)[name = tensor("clip_31")]; - tensor x_47_axes_0 = const()[name = tensor("x_47_axes_0"), val = tensor([-1])]; - tensor x_47 = layer_norm(axes = x_47_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_261)[name = tensor("x_47")]; - tensor input_263_perm_0 = const()[name = tensor("input_263_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_105_pad_type_0 = const()[name = tensor("inputs_105_pad_type_0"), val = tensor("valid")]; - tensor inputs_105_strides_0 = const()[name = tensor("inputs_105_strides_0"), val = tensor([1])]; - tensor inputs_105_pad_0 = const()[name = tensor("inputs_105_pad_0"), val = tensor([0, 0])]; - tensor inputs_105_dilations_0 = const()[name = tensor("inputs_105_dilations_0"), val = tensor([1])]; - tensor inputs_105_groups_0 = const()[name = tensor("inputs_105_groups_0"), val = tensor(1)]; - tensor input_263 = transpose(perm = input_263_perm_0, x = x_47)[name = tensor("transpose_40")]; - tensor inputs_105 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_105_dilations_0, groups = inputs_105_groups_0, pad = inputs_105_pad_0, pad_type = inputs_105_pad_type_0, strides = inputs_105_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_263)[name = tensor("inputs_105")]; - tensor var_1923_split_sizes_0 = const()[name = tensor("op_1923_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1923_axis_0 = const()[name = tensor("op_1923_axis_0"), val = tensor(1)]; - tensor var_1923_0, tensor var_1923_1 = split(axis = var_1923_axis_0, split_sizes = var_1923_split_sizes_0, x = inputs_105)[name = tensor("op_1923")]; - tensor var_1925 = sigmoid(x = var_1923_1)[name = tensor("op_1925")]; - tensor inputs_107 = mul(x = var_1923_0, y = var_1925)[name = tensor("inputs_107")]; - tensor outputs_aug_31_pad_type_0 = const()[name = tensor("outputs_aug_31_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_31_pad_0 = const()[name = tensor("outputs_aug_31_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_31_groups_0 = const()[name = tensor("outputs_aug_31_groups_0"), val = tensor(256)]; - tensor outputs_aug_31_strides_0 = const()[name = tensor("outputs_aug_31_strides_0"), val = tensor([1])]; - tensor outputs_aug_31_dilations_0 = const()[name = tensor("outputs_aug_31_dilations_0"), val = tensor([1])]; - tensor outputs_aug_31 = conv(dilations = outputs_aug_31_dilations_0, groups = outputs_aug_31_groups_0, pad = outputs_aug_31_pad_0, pad_type = outputs_aug_31_pad_type_0, strides = outputs_aug_31_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_107)[name = tensor("outputs_aug_31")]; - tensor input_265_begin_0 = const()[name = tensor("input_265_begin_0"), val = tensor([0, 0, 0])]; - tensor input_265_end_0 = const()[name = tensor("input_265_end_0"), val = tensor([1, 256, 16])]; - tensor input_265_end_mask_0 = const()[name = tensor("input_265_end_mask_0"), val = tensor([true, true, false])]; - tensor input_265 = slice_by_index(begin = input_265_begin_0, end = input_265_end_0, end_mask = input_265_end_mask_0, x = outputs_aug_31)[name = tensor("input_265")]; - tensor inputs_109 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_265)[name = tensor("inputs_109")]; - tensor input_267 = silu(x = inputs_109)[name = tensor("input_267")]; - tensor input_269_pad_type_0 = const()[name = tensor("input_269_pad_type_0"), val = tensor("valid")]; - tensor input_269_strides_0 = const()[name = tensor("input_269_strides_0"), val = tensor([1])]; - tensor input_269_pad_0 = const()[name = tensor("input_269_pad_0"), val = tensor([0, 0])]; - tensor input_269_dilations_0 = const()[name = tensor("input_269_dilations_0"), val = tensor([1])]; - tensor input_269_groups_0 = const()[name = tensor("input_269_groups_0"), val = tensor(1)]; - tensor input_269 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_269_dilations_0, groups = input_269_groups_0, pad = input_269_pad_0, pad_type = input_269_pad_type_0, strides = input_269_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_267)[name = tensor("input_269")]; - tensor conv_out_31_perm_0 = const()[name = tensor("conv_out_31_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1956_begin_0 = const()[name = tensor("op_1956_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1956_end_0 = const()[name = tensor("op_1956_end_0"), val = tensor([1, 16, 256])]; - tensor var_1956_end_mask_0 = const()[name = tensor("op_1956_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_31 = transpose(perm = conv_out_31_perm_0, x = input_269)[name = tensor("transpose_39")]; - tensor var_1956 = slice_by_index(begin = var_1956_begin_0, end = var_1956_end_0, end_mask = var_1956_end_mask_0, x = conv_out_31)[name = tensor("op_1956")]; - tensor var_1959_begin_0 = const()[name = tensor("op_1959_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1959_end_0 = const()[name = tensor("op_1959_end_0"), val = tensor([1, 2, 256])]; - tensor var_1959_end_mask_0 = const()[name = tensor("op_1959_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1959 = slice_by_index(begin = var_1959_begin_0, end = var_1959_end_0, end_mask = var_1959_end_mask_0, x = x_45)[name = tensor("op_1959")]; - tensor var_1962_begin_0 = const()[name = tensor("op_1962_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1962_end_0 = const()[name = tensor("op_1962_end_0"), val = tensor([1, 16, 256])]; - tensor var_1962_end_mask_0 = const()[name = tensor("op_1962_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1962 = slice_by_index(begin = var_1962_begin_0, end = var_1962_end_0, end_mask = var_1962_end_mask_0, x = input_261)[name = tensor("op_1962")]; + tensor x_expanded_31 = tile(reps = x_expanded_31_reps_0, x = var_1230)[name = tensor("x_expanded_31")]; + tensor var_1244 = sub(x = var_18, y = fill_mask_31)[name = tensor("op_1244")]; + tensor var_1245 = mul(x = window_37, y = var_1244)[name = tensor("op_1245")]; + tensor var_1246 = mul(x = x_expanded_31, y = fill_mask_31)[name = tensor("op_1246")]; + tensor filled_window_31 = add(x = var_1245, y = var_1246)[name = tensor("filled_window_31")]; + tensor var_1248 = greater_equal(x = filled_31, y = var_17)[name = tensor("op_1248")]; + tensor var_1249_dtype_0 = const()[name = tensor("op_1249_dtype_0"), val = tensor("fp32")]; + tensor var_1250 = const()[name = tensor("op_1250"), val = tensor([1, 1, 1])]; + tensor var_1249 = cast(dtype = var_1249_dtype_0, x = var_1248)[name = tensor("cast_160")]; + tensor use_roll_31 = reshape(shape = var_1250, x = var_1249)[name = tensor("use_roll_31")]; + tensor var_1252 = sub(x = var_18, y = use_roll_31)[name = tensor("op_1252")]; + tensor var_1253 = mul(x = filled_window_31, y = var_1252)[name = tensor("op_1253")]; + tensor var_1254 = mul(x = rolled_31, y = use_roll_31)[name = tensor("op_1254")]; + tensor window_39 = add(x = var_1253, y = var_1254)[name = tensor("window_39")]; + tensor var_1256 = const()[name = tensor("op_1256"), val = tensor(0x1p+0)]; + tensor var_1257 = add(x = filled_31, y = var_1256)[name = tensor("op_1257")]; + tensor clip_31 = clip(alpha = var_22, beta = var_17, x = var_1257)[name = tensor("clip_31")]; + tensor var_1260_begin_0 = const()[name = tensor("op_1260_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1260_end_0 = const()[name = tensor("op_1260_end_0"), val = tensor([1, 2, 256])]; + tensor var_1260_end_mask_0 = const()[name = tensor("op_1260_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1260 = slice_by_index(begin = var_1260_begin_0, end = var_1260_end_0, end_mask = var_1260_end_mask_0, x = x_21)[name = tensor("op_1260")]; + tensor var_1263_begin_0 = const()[name = tensor("op_1263_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1263_end_0 = const()[name = tensor("op_1263_end_0"), val = tensor([1, 16, 256])]; + tensor var_1263_end_mask_0 = const()[name = tensor("op_1263_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1263 = slice_by_index(begin = var_1263_begin_0, end = var_1263_end_0, end_mask = var_1263_end_mask_0, x = window_39)[name = tensor("op_1263")]; tensor rolled_33_interleave_0 = const()[name = tensor("rolled_33_interleave_0"), val = tensor(false)]; - tensor rolled_33 = concat(axis = var_34, interleave = rolled_33_interleave_0, values = (var_1962, var_1959))[name = tensor("rolled_33")]; + tensor rolled_33 = concat(axis = var_34, interleave = rolled_33_interleave_0, values = (var_1263, var_1260))[name = tensor("rolled_33")]; tensor clip_32 = clip(alpha = var_22, beta = var_21, x = clip_31)[name = tensor("clip_32")]; - tensor var_1967 = equal(x = encoder__conv_positions, y = clip_32)[name = tensor("op_1967")]; - tensor var_1968_dtype_0 = const()[name = tensor("op_1968_dtype_0"), val = tensor("fp32")]; - tensor var_1969 = const()[name = tensor("op_1969"), val = tensor([1, 16, 1])]; - tensor var_1968 = cast(dtype = var_1968_dtype_0, x = var_1967)[name = tensor("cast_159")]; - tensor fill_mask_33 = reshape(shape = var_1969, x = var_1968)[name = tensor("fill_mask_33")]; + tensor var_1268 = equal(x = encoder__conv_positions, y = clip_32)[name = tensor("op_1268")]; + tensor var_1269_dtype_0 = const()[name = tensor("op_1269_dtype_0"), val = tensor("fp32")]; + tensor var_1270 = const()[name = tensor("op_1270"), val = tensor([1, 16, 1])]; + tensor var_1269 = cast(dtype = var_1269_dtype_0, x = var_1268)[name = tensor("cast_159")]; + tensor fill_mask_33 = reshape(shape = var_1270, x = var_1269)[name = tensor("fill_mask_33")]; tensor x_expanded_33_reps_0 = const()[name = tensor("x_expanded_33_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_33 = tile(reps = x_expanded_33_reps_0, x = var_1959)[name = tensor("x_expanded_33")]; - tensor var_1973 = sub(x = var_18, y = fill_mask_33)[name = tensor("op_1973")]; - tensor var_1974 = mul(x = input_261, y = var_1973)[name = tensor("op_1974")]; - tensor var_1975 = mul(x = x_expanded_33, y = fill_mask_33)[name = tensor("op_1975")]; - tensor filled_window_33 = add(x = var_1974, y = var_1975)[name = tensor("filled_window_33")]; - tensor var_1977 = greater_equal(x = clip_31, y = var_17)[name = tensor("op_1977")]; - tensor var_1978_dtype_0 = const()[name = tensor("op_1978_dtype_0"), val = tensor("fp32")]; - tensor var_1979 = const()[name = tensor("op_1979"), val = tensor([1, 1, 1])]; - tensor var_1978 = cast(dtype = var_1978_dtype_0, x = var_1977)[name = tensor("cast_158")]; - tensor use_roll_33 = reshape(shape = var_1979, x = var_1978)[name = tensor("use_roll_33")]; - tensor var_1981 = sub(x = var_18, y = use_roll_33)[name = tensor("op_1981")]; - tensor var_1982 = mul(x = filled_window_33, y = var_1981)[name = tensor("op_1982")]; - tensor var_1983 = mul(x = rolled_33, y = use_roll_33)[name = tensor("op_1983")]; - tensor input_271 = add(x = var_1982, y = var_1983)[name = tensor("input_271")]; - tensor var_1985 = const()[name = tensor("op_1985"), val = tensor(0x1p+0)]; - tensor var_1986 = add(x = clip_31, y = var_1985)[name = tensor("op_1986")]; - tensor clip_33 = clip(alpha = var_22, beta = var_17, x = var_1986)[name = tensor("clip_33")]; - tensor x_49_axes_0 = const()[name = tensor("x_49_axes_0"), val = tensor([-1])]; - tensor x_49 = layer_norm(axes = x_49_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_271)[name = tensor("x_49")]; - tensor input_273_perm_0 = const()[name = tensor("input_273_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_111_pad_type_0 = const()[name = tensor("inputs_111_pad_type_0"), val = tensor("valid")]; - tensor inputs_111_strides_0 = const()[name = tensor("inputs_111_strides_0"), val = tensor([1])]; - tensor inputs_111_pad_0 = const()[name = tensor("inputs_111_pad_0"), val = tensor([0, 0])]; - tensor inputs_111_dilations_0 = const()[name = tensor("inputs_111_dilations_0"), val = tensor([1])]; - tensor inputs_111_groups_0 = const()[name = tensor("inputs_111_groups_0"), val = tensor(1)]; - tensor input_273 = transpose(perm = input_273_perm_0, x = x_49)[name = tensor("transpose_38")]; - tensor inputs_111 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_111_dilations_0, groups = inputs_111_groups_0, pad = inputs_111_pad_0, pad_type = inputs_111_pad_type_0, strides = inputs_111_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_273)[name = tensor("inputs_111")]; - tensor var_2007_split_sizes_0 = const()[name = tensor("op_2007_split_sizes_0"), val = tensor([256, 256])]; - tensor var_2007_axis_0 = const()[name = tensor("op_2007_axis_0"), val = tensor(1)]; - tensor var_2007_0, tensor var_2007_1 = split(axis = var_2007_axis_0, split_sizes = var_2007_split_sizes_0, x = inputs_111)[name = tensor("op_2007")]; - tensor var_2009 = sigmoid(x = var_2007_1)[name = tensor("op_2009")]; - tensor inputs_113 = mul(x = var_2007_0, y = var_2009)[name = tensor("inputs_113")]; - tensor outputs_aug_33_pad_type_0 = const()[name = tensor("outputs_aug_33_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_33_pad_0 = const()[name = tensor("outputs_aug_33_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_33_groups_0 = const()[name = tensor("outputs_aug_33_groups_0"), val = tensor(256)]; - tensor outputs_aug_33_strides_0 = const()[name = tensor("outputs_aug_33_strides_0"), val = tensor([1])]; - tensor outputs_aug_33_dilations_0 = const()[name = tensor("outputs_aug_33_dilations_0"), val = tensor([1])]; - tensor outputs_aug_33 = conv(dilations = outputs_aug_33_dilations_0, groups = outputs_aug_33_groups_0, pad = outputs_aug_33_pad_0, pad_type = outputs_aug_33_pad_type_0, strides = outputs_aug_33_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_113)[name = tensor("outputs_aug_33")]; - tensor input_275_begin_0 = const()[name = tensor("input_275_begin_0"), val = tensor([0, 0, 0])]; - tensor input_275_end_0 = const()[name = tensor("input_275_end_0"), val = tensor([1, 256, 16])]; - tensor input_275_end_mask_0 = const()[name = tensor("input_275_end_mask_0"), val = tensor([true, true, false])]; - tensor input_275 = slice_by_index(begin = input_275_begin_0, end = input_275_end_0, end_mask = input_275_end_mask_0, x = outputs_aug_33)[name = tensor("input_275")]; - tensor inputs_115 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_275)[name = tensor("inputs_115")]; - tensor input_277 = silu(x = inputs_115)[name = tensor("input_277")]; - tensor input_279_pad_type_0 = const()[name = tensor("input_279_pad_type_0"), val = tensor("valid")]; - tensor input_279_strides_0 = const()[name = tensor("input_279_strides_0"), val = tensor([1])]; - tensor input_279_pad_0 = const()[name = tensor("input_279_pad_0"), val = tensor([0, 0])]; - tensor input_279_dilations_0 = const()[name = tensor("input_279_dilations_0"), val = tensor([1])]; - tensor input_279_groups_0 = const()[name = tensor("input_279_groups_0"), val = tensor(1)]; - tensor input_279 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_279_dilations_0, groups = input_279_groups_0, pad = input_279_pad_0, pad_type = input_279_pad_type_0, strides = input_279_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_277)[name = tensor("input_279")]; - tensor conv_out_33_perm_0 = const()[name = tensor("conv_out_33_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2040_begin_0 = const()[name = tensor("op_2040_begin_0"), val = tensor([0, -1, 0])]; - tensor var_2040_end_0 = const()[name = tensor("op_2040_end_0"), val = tensor([1, 16, 256])]; - tensor var_2040_end_mask_0 = const()[name = tensor("op_2040_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_33 = transpose(perm = conv_out_33_perm_0, x = input_279)[name = tensor("transpose_37")]; - tensor var_2040 = slice_by_index(begin = var_2040_begin_0, end = var_2040_end_0, end_mask = var_2040_end_mask_0, x = conv_out_33)[name = tensor("op_2040")]; - tensor var_2043_begin_0 = const()[name = tensor("op_2043_begin_0"), val = tensor([0, 2, 0])]; - tensor var_2043_end_0 = const()[name = tensor("op_2043_end_0"), val = tensor([1, 3, 256])]; - tensor var_2043_end_mask_0 = const()[name = tensor("op_2043_end_mask_0"), val = tensor([true, false, true])]; - tensor var_2043 = slice_by_index(begin = var_2043_begin_0, end = var_2043_end_0, end_mask = var_2043_end_mask_0, x = x_45)[name = tensor("op_2043")]; - tensor var_2046_begin_0 = const()[name = tensor("op_2046_begin_0"), val = tensor([0, 1, 0])]; - tensor var_2046_end_0 = const()[name = tensor("op_2046_end_0"), val = tensor([1, 16, 256])]; - tensor var_2046_end_mask_0 = const()[name = tensor("op_2046_end_mask_0"), val = tensor([true, true, true])]; - tensor var_2046 = slice_by_index(begin = var_2046_begin_0, end = var_2046_end_0, end_mask = var_2046_end_mask_0, x = input_271)[name = tensor("op_2046")]; + tensor x_expanded_33 = tile(reps = x_expanded_33_reps_0, x = var_1260)[name = tensor("x_expanded_33")]; + tensor var_1274 = sub(x = var_18, y = fill_mask_33)[name = tensor("op_1274")]; + tensor var_1275 = mul(x = window_39, y = var_1274)[name = tensor("op_1275")]; + tensor var_1276 = mul(x = x_expanded_33, y = fill_mask_33)[name = tensor("op_1276")]; + tensor filled_window_33 = add(x = var_1275, y = var_1276)[name = tensor("filled_window_33")]; + tensor var_1278 = greater_equal(x = clip_31, y = var_17)[name = tensor("op_1278")]; + tensor var_1279_dtype_0 = const()[name = tensor("op_1279_dtype_0"), val = tensor("fp32")]; + tensor var_1280 = const()[name = tensor("op_1280"), val = tensor([1, 1, 1])]; + tensor var_1279 = cast(dtype = var_1279_dtype_0, x = var_1278)[name = tensor("cast_158")]; + tensor use_roll_33 = reshape(shape = var_1280, x = var_1279)[name = tensor("use_roll_33")]; + tensor var_1282 = sub(x = var_18, y = use_roll_33)[name = tensor("op_1282")]; + tensor var_1283 = mul(x = filled_window_33, y = var_1282)[name = tensor("op_1283")]; + tensor var_1284 = mul(x = rolled_33, y = use_roll_33)[name = tensor("op_1284")]; + tensor window_41 = add(x = var_1283, y = var_1284)[name = tensor("window_41")]; + tensor var_1286 = const()[name = tensor("op_1286"), val = tensor(0x1p+0)]; + tensor var_1287 = add(x = clip_31, y = var_1286)[name = tensor("op_1287")]; + tensor clip_33 = clip(alpha = var_22, beta = var_17, x = var_1287)[name = tensor("clip_33")]; + tensor var_1290_begin_0 = const()[name = tensor("op_1290_begin_0"), val = tensor([0, 2, 0])]; + tensor var_1290_end_0 = const()[name = tensor("op_1290_end_0"), val = tensor([1, 3, 256])]; + tensor var_1290_end_mask_0 = const()[name = tensor("op_1290_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1290 = slice_by_index(begin = var_1290_begin_0, end = var_1290_end_0, end_mask = var_1290_end_mask_0, x = x_21)[name = tensor("op_1290")]; + tensor var_1293_begin_0 = const()[name = tensor("op_1293_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1293_end_0 = const()[name = tensor("op_1293_end_0"), val = tensor([1, 16, 256])]; + tensor var_1293_end_mask_0 = const()[name = tensor("op_1293_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1293 = slice_by_index(begin = var_1293_begin_0, end = var_1293_end_0, end_mask = var_1293_end_mask_0, x = window_41)[name = tensor("op_1293")]; tensor rolled_35_interleave_0 = const()[name = tensor("rolled_35_interleave_0"), val = tensor(false)]; - tensor rolled_35 = concat(axis = var_34, interleave = rolled_35_interleave_0, values = (var_2046, var_2043))[name = tensor("rolled_35")]; + tensor rolled_35 = concat(axis = var_34, interleave = rolled_35_interleave_0, values = (var_1293, var_1290))[name = tensor("rolled_35")]; tensor clip_34 = clip(alpha = var_22, beta = var_21, x = clip_33)[name = tensor("clip_34")]; - tensor var_2051 = equal(x = encoder__conv_positions, y = clip_34)[name = tensor("op_2051")]; - tensor var_2052_dtype_0 = const()[name = tensor("op_2052_dtype_0"), val = tensor("fp32")]; - tensor var_2053 = const()[name = tensor("op_2053"), val = tensor([1, 16, 1])]; - tensor var_2052 = cast(dtype = var_2052_dtype_0, x = var_2051)[name = tensor("cast_157")]; - tensor fill_mask_35 = reshape(shape = var_2053, x = var_2052)[name = tensor("fill_mask_35")]; + tensor var_1298 = equal(x = encoder__conv_positions, y = clip_34)[name = tensor("op_1298")]; + tensor var_1299_dtype_0 = const()[name = tensor("op_1299_dtype_0"), val = tensor("fp32")]; + tensor var_1300 = const()[name = tensor("op_1300"), val = tensor([1, 16, 1])]; + tensor var_1299 = cast(dtype = var_1299_dtype_0, x = var_1298)[name = tensor("cast_157")]; + tensor fill_mask_35 = reshape(shape = var_1300, x = var_1299)[name = tensor("fill_mask_35")]; tensor x_expanded_35_reps_0 = const()[name = tensor("x_expanded_35_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_35 = tile(reps = x_expanded_35_reps_0, x = var_2043)[name = tensor("x_expanded_35")]; - tensor var_2057 = sub(x = var_18, y = fill_mask_35)[name = tensor("op_2057")]; - tensor var_2058 = mul(x = input_271, y = var_2057)[name = tensor("op_2058")]; - tensor var_2059 = mul(x = x_expanded_35, y = fill_mask_35)[name = tensor("op_2059")]; - tensor filled_window_35 = add(x = var_2058, y = var_2059)[name = tensor("filled_window_35")]; - tensor var_2061 = greater_equal(x = clip_33, y = var_17)[name = tensor("op_2061")]; - tensor var_2062_dtype_0 = const()[name = tensor("op_2062_dtype_0"), val = tensor("fp32")]; - tensor var_2063 = const()[name = tensor("op_2063"), val = tensor([1, 1, 1])]; - tensor var_2062 = cast(dtype = var_2062_dtype_0, x = var_2061)[name = tensor("cast_156")]; - tensor use_roll_35 = reshape(shape = var_2063, x = var_2062)[name = tensor("use_roll_35")]; - tensor var_2065 = sub(x = var_18, y = use_roll_35)[name = tensor("op_2065")]; - tensor var_2066 = mul(x = filled_window_35, y = var_2065)[name = tensor("op_2066")]; - tensor var_2067 = mul(x = rolled_35, y = use_roll_35)[name = tensor("op_2067")]; - tensor input_281 = add(x = var_2066, y = var_2067)[name = tensor("input_281")]; - tensor var_2069 = const()[name = tensor("op_2069"), val = tensor(0x1p+0)]; - tensor var_2070 = add(x = clip_33, y = var_2069)[name = tensor("op_2070")]; - tensor clip_35 = clip(alpha = var_22, beta = var_17, x = var_2070)[name = tensor("clip_35")]; - tensor x_51_axes_0 = const()[name = tensor("x_51_axes_0"), val = tensor([-1])]; - tensor x_51 = layer_norm(axes = x_51_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_281)[name = tensor("x_51")]; - tensor input_283_perm_0 = const()[name = tensor("input_283_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_117_pad_type_0 = const()[name = tensor("inputs_117_pad_type_0"), val = tensor("valid")]; - tensor inputs_117_strides_0 = const()[name = tensor("inputs_117_strides_0"), val = tensor([1])]; - tensor inputs_117_pad_0 = const()[name = tensor("inputs_117_pad_0"), val = tensor([0, 0])]; - tensor inputs_117_dilations_0 = const()[name = tensor("inputs_117_dilations_0"), val = tensor([1])]; - tensor inputs_117_groups_0 = const()[name = tensor("inputs_117_groups_0"), val = tensor(1)]; - tensor input_283 = transpose(perm = input_283_perm_0, x = x_51)[name = tensor("transpose_36")]; - tensor inputs_117 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_117_dilations_0, groups = inputs_117_groups_0, pad = inputs_117_pad_0, pad_type = inputs_117_pad_type_0, strides = inputs_117_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_283)[name = tensor("inputs_117")]; - tensor var_2091_split_sizes_0 = const()[name = tensor("op_2091_split_sizes_0"), val = tensor([256, 256])]; - tensor var_2091_axis_0 = const()[name = tensor("op_2091_axis_0"), val = tensor(1)]; - tensor var_2091_0, tensor var_2091_1 = split(axis = var_2091_axis_0, split_sizes = var_2091_split_sizes_0, x = inputs_117)[name = tensor("op_2091")]; - tensor var_2093 = sigmoid(x = var_2091_1)[name = tensor("op_2093")]; - tensor inputs_119 = mul(x = var_2091_0, y = var_2093)[name = tensor("inputs_119")]; - tensor outputs_aug_35_pad_type_0 = const()[name = tensor("outputs_aug_35_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_35_pad_0 = const()[name = tensor("outputs_aug_35_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_35_groups_0 = const()[name = tensor("outputs_aug_35_groups_0"), val = tensor(256)]; - tensor outputs_aug_35_strides_0 = const()[name = tensor("outputs_aug_35_strides_0"), val = tensor([1])]; - tensor outputs_aug_35_dilations_0 = const()[name = tensor("outputs_aug_35_dilations_0"), val = tensor([1])]; - tensor outputs_aug_35 = conv(dilations = outputs_aug_35_dilations_0, groups = outputs_aug_35_groups_0, pad = outputs_aug_35_pad_0, pad_type = outputs_aug_35_pad_type_0, strides = outputs_aug_35_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_119)[name = tensor("outputs_aug_35")]; - tensor input_285_begin_0 = const()[name = tensor("input_285_begin_0"), val = tensor([0, 0, 0])]; - tensor input_285_end_0 = const()[name = tensor("input_285_end_0"), val = tensor([1, 256, 16])]; - tensor input_285_end_mask_0 = const()[name = tensor("input_285_end_mask_0"), val = tensor([true, true, false])]; - tensor input_285 = slice_by_index(begin = input_285_begin_0, end = input_285_end_0, end_mask = input_285_end_mask_0, x = outputs_aug_35)[name = tensor("input_285")]; - tensor inputs_121 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_285)[name = tensor("inputs_121")]; - tensor input_287 = silu(x = inputs_121)[name = tensor("input_287")]; - tensor input_289_pad_type_0 = const()[name = tensor("input_289_pad_type_0"), val = tensor("valid")]; - tensor input_289_strides_0 = const()[name = tensor("input_289_strides_0"), val = tensor([1])]; - tensor input_289_pad_0 = const()[name = tensor("input_289_pad_0"), val = tensor([0, 0])]; - tensor input_289_dilations_0 = const()[name = tensor("input_289_dilations_0"), val = tensor([1])]; - tensor input_289_groups_0 = const()[name = tensor("input_289_groups_0"), val = tensor(1)]; - tensor input_289 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_289_dilations_0, groups = input_289_groups_0, pad = input_289_pad_0, pad_type = input_289_pad_type_0, strides = input_289_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_287)[name = tensor("input_289")]; - tensor conv_out_35_perm_0 = const()[name = tensor("conv_out_35_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2124_begin_0 = const()[name = tensor("op_2124_begin_0"), val = tensor([0, -1, 0])]; - tensor var_2124_end_0 = const()[name = tensor("op_2124_end_0"), val = tensor([1, 16, 256])]; - tensor var_2124_end_mask_0 = const()[name = tensor("op_2124_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_35 = transpose(perm = conv_out_35_perm_0, x = input_289)[name = tensor("transpose_35")]; - tensor var_2124 = slice_by_index(begin = var_2124_begin_0, end = var_2124_end_0, end_mask = var_2124_end_mask_0, x = conv_out_35)[name = tensor("op_2124")]; - tensor var_2127_begin_0 = const()[name = tensor("op_2127_begin_0"), val = tensor([0, 3, 0])]; - tensor var_2127_end_0 = const()[name = tensor("op_2127_end_0"), val = tensor([1, 4, 256])]; - tensor var_2127_end_mask_0 = const()[name = tensor("op_2127_end_mask_0"), val = tensor([true, false, true])]; - tensor var_2127 = slice_by_index(begin = var_2127_begin_0, end = var_2127_end_0, end_mask = var_2127_end_mask_0, x = x_45)[name = tensor("op_2127")]; - tensor var_2130_begin_0 = const()[name = tensor("op_2130_begin_0"), val = tensor([0, 1, 0])]; - tensor var_2130_end_0 = const()[name = tensor("op_2130_end_0"), val = tensor([1, 16, 256])]; - tensor var_2130_end_mask_0 = const()[name = tensor("op_2130_end_mask_0"), val = tensor([true, true, true])]; - tensor var_2130 = slice_by_index(begin = var_2130_begin_0, end = var_2130_end_0, end_mask = var_2130_end_mask_0, x = input_281)[name = tensor("op_2130")]; + tensor x_expanded_35 = tile(reps = x_expanded_35_reps_0, x = var_1290)[name = tensor("x_expanded_35")]; + tensor var_1304 = sub(x = var_18, y = fill_mask_35)[name = tensor("op_1304")]; + tensor var_1305 = mul(x = window_41, y = var_1304)[name = tensor("op_1305")]; + tensor var_1306 = mul(x = x_expanded_35, y = fill_mask_35)[name = tensor("op_1306")]; + tensor filled_window_35 = add(x = var_1305, y = var_1306)[name = tensor("filled_window_35")]; + tensor var_1308 = greater_equal(x = clip_33, y = var_17)[name = tensor("op_1308")]; + tensor var_1309_dtype_0 = const()[name = tensor("op_1309_dtype_0"), val = tensor("fp32")]; + tensor var_1310 = const()[name = tensor("op_1310"), val = tensor([1, 1, 1])]; + tensor var_1309 = cast(dtype = var_1309_dtype_0, x = var_1308)[name = tensor("cast_156")]; + tensor use_roll_35 = reshape(shape = var_1310, x = var_1309)[name = tensor("use_roll_35")]; + tensor var_1312 = sub(x = var_18, y = use_roll_35)[name = tensor("op_1312")]; + tensor var_1313 = mul(x = filled_window_35, y = var_1312)[name = tensor("op_1313")]; + tensor var_1314 = mul(x = rolled_35, y = use_roll_35)[name = tensor("op_1314")]; + tensor window_43 = add(x = var_1313, y = var_1314)[name = tensor("window_43")]; + tensor var_1316 = const()[name = tensor("op_1316"), val = tensor(0x1p+0)]; + tensor var_1317 = add(x = clip_33, y = var_1316)[name = tensor("op_1317")]; + tensor clip_35 = clip(alpha = var_22, beta = var_17, x = var_1317)[name = tensor("clip_35")]; + tensor var_1320_begin_0 = const()[name = tensor("op_1320_begin_0"), val = tensor([0, 3, 0])]; + tensor var_1320_end_0 = const()[name = tensor("op_1320_end_0"), val = tensor([1, 4, 256])]; + tensor var_1320_end_mask_0 = const()[name = tensor("op_1320_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1320 = slice_by_index(begin = var_1320_begin_0, end = var_1320_end_0, end_mask = var_1320_end_mask_0, x = x_21)[name = tensor("op_1320")]; + tensor var_1323_begin_0 = const()[name = tensor("op_1323_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1323_end_0 = const()[name = tensor("op_1323_end_0"), val = tensor([1, 16, 256])]; + tensor var_1323_end_mask_0 = const()[name = tensor("op_1323_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1323 = slice_by_index(begin = var_1323_begin_0, end = var_1323_end_0, end_mask = var_1323_end_mask_0, x = window_43)[name = tensor("op_1323")]; tensor rolled_37_interleave_0 = const()[name = tensor("rolled_37_interleave_0"), val = tensor(false)]; - tensor rolled_37 = concat(axis = var_34, interleave = rolled_37_interleave_0, values = (var_2130, var_2127))[name = tensor("rolled_37")]; + tensor rolled_37 = concat(axis = var_34, interleave = rolled_37_interleave_0, values = (var_1323, var_1320))[name = tensor("rolled_37")]; tensor clip_36 = clip(alpha = var_22, beta = var_21, x = clip_35)[name = tensor("clip_36")]; - tensor var_2135 = equal(x = encoder__conv_positions, y = clip_36)[name = tensor("op_2135")]; - tensor var_2136_dtype_0 = const()[name = tensor("op_2136_dtype_0"), val = tensor("fp32")]; - tensor var_2137 = const()[name = tensor("op_2137"), val = tensor([1, 16, 1])]; - tensor var_2136 = cast(dtype = var_2136_dtype_0, x = var_2135)[name = tensor("cast_155")]; - tensor fill_mask_37 = reshape(shape = var_2137, x = var_2136)[name = tensor("fill_mask_37")]; + tensor var_1328 = equal(x = encoder__conv_positions, y = clip_36)[name = tensor("op_1328")]; + tensor var_1329_dtype_0 = const()[name = tensor("op_1329_dtype_0"), val = tensor("fp32")]; + tensor var_1330 = const()[name = tensor("op_1330"), val = tensor([1, 16, 1])]; + tensor var_1329 = cast(dtype = var_1329_dtype_0, x = var_1328)[name = tensor("cast_155")]; + tensor fill_mask_37 = reshape(shape = var_1330, x = var_1329)[name = tensor("fill_mask_37")]; tensor x_expanded_37_reps_0 = const()[name = tensor("x_expanded_37_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_37 = tile(reps = x_expanded_37_reps_0, x = var_2127)[name = tensor("x_expanded_37")]; - tensor var_2141 = sub(x = var_18, y = fill_mask_37)[name = tensor("op_2141")]; - tensor var_2142 = mul(x = input_281, y = var_2141)[name = tensor("op_2142")]; - tensor var_2143 = mul(x = x_expanded_37, y = fill_mask_37)[name = tensor("op_2143")]; - tensor filled_window_37 = add(x = var_2142, y = var_2143)[name = tensor("filled_window_37")]; - tensor var_2145 = greater_equal(x = clip_35, y = var_17)[name = tensor("op_2145")]; - tensor var_2146_dtype_0 = const()[name = tensor("op_2146_dtype_0"), val = tensor("fp32")]; - tensor var_2147 = const()[name = tensor("op_2147"), val = tensor([1, 1, 1])]; - tensor var_2146 = cast(dtype = var_2146_dtype_0, x = var_2145)[name = tensor("cast_154")]; - tensor use_roll_37 = reshape(shape = var_2147, x = var_2146)[name = tensor("use_roll_37")]; - tensor var_2149 = sub(x = var_18, y = use_roll_37)[name = tensor("op_2149")]; - tensor var_2150 = mul(x = filled_window_37, y = var_2149)[name = tensor("op_2150")]; - tensor var_2151 = mul(x = rolled_37, y = use_roll_37)[name = tensor("op_2151")]; - tensor input_291 = add(x = var_2150, y = var_2151)[name = tensor("input_291")]; - tensor var_2153 = const()[name = tensor("op_2153"), val = tensor(0x1p+0)]; - tensor var_2154 = add(x = clip_35, y = var_2153)[name = tensor("op_2154")]; - tensor clip_37 = clip(alpha = var_22, beta = var_17, x = var_2154)[name = tensor("clip_37")]; - tensor x_53_axes_0 = const()[name = tensor("x_53_axes_0"), val = tensor([-1])]; - tensor x_53 = layer_norm(axes = x_53_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_291)[name = tensor("x_53")]; - tensor input_293_perm_0 = const()[name = tensor("input_293_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_123_pad_type_0 = const()[name = tensor("inputs_123_pad_type_0"), val = tensor("valid")]; - tensor inputs_123_strides_0 = const()[name = tensor("inputs_123_strides_0"), val = tensor([1])]; - tensor inputs_123_pad_0 = const()[name = tensor("inputs_123_pad_0"), val = tensor([0, 0])]; - tensor inputs_123_dilations_0 = const()[name = tensor("inputs_123_dilations_0"), val = tensor([1])]; - tensor inputs_123_groups_0 = const()[name = tensor("inputs_123_groups_0"), val = tensor(1)]; - tensor input_293 = transpose(perm = input_293_perm_0, x = x_53)[name = tensor("transpose_34")]; - tensor inputs_123 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_123_dilations_0, groups = inputs_123_groups_0, pad = inputs_123_pad_0, pad_type = inputs_123_pad_type_0, strides = inputs_123_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_293)[name = tensor("inputs_123")]; - tensor var_2175_split_sizes_0 = const()[name = tensor("op_2175_split_sizes_0"), val = tensor([256, 256])]; - tensor var_2175_axis_0 = const()[name = tensor("op_2175_axis_0"), val = tensor(1)]; - tensor var_2175_0, tensor var_2175_1 = split(axis = var_2175_axis_0, split_sizes = var_2175_split_sizes_0, x = inputs_123)[name = tensor("op_2175")]; - tensor var_2177 = sigmoid(x = var_2175_1)[name = tensor("op_2177")]; - tensor inputs_125 = mul(x = var_2175_0, y = var_2177)[name = tensor("inputs_125")]; - tensor outputs_aug_37_pad_type_0 = const()[name = tensor("outputs_aug_37_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_37_pad_0 = const()[name = tensor("outputs_aug_37_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_37_groups_0 = const()[name = tensor("outputs_aug_37_groups_0"), val = tensor(256)]; - tensor outputs_aug_37_strides_0 = const()[name = tensor("outputs_aug_37_strides_0"), val = tensor([1])]; - tensor outputs_aug_37_dilations_0 = const()[name = tensor("outputs_aug_37_dilations_0"), val = tensor([1])]; - tensor outputs_aug_37 = conv(dilations = outputs_aug_37_dilations_0, groups = outputs_aug_37_groups_0, pad = outputs_aug_37_pad_0, pad_type = outputs_aug_37_pad_type_0, strides = outputs_aug_37_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_125)[name = tensor("outputs_aug_37")]; - tensor input_295_begin_0 = const()[name = tensor("input_295_begin_0"), val = tensor([0, 0, 0])]; - tensor input_295_end_0 = const()[name = tensor("input_295_end_0"), val = tensor([1, 256, 16])]; - tensor input_295_end_mask_0 = const()[name = tensor("input_295_end_mask_0"), val = tensor([true, true, false])]; - tensor input_295 = slice_by_index(begin = input_295_begin_0, end = input_295_end_0, end_mask = input_295_end_mask_0, x = outputs_aug_37)[name = tensor("input_295")]; - tensor inputs_127 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_295)[name = tensor("inputs_127")]; - tensor input_297 = silu(x = inputs_127)[name = tensor("input_297")]; - tensor input_299_pad_type_0 = const()[name = tensor("input_299_pad_type_0"), val = tensor("valid")]; - tensor input_299_strides_0 = const()[name = tensor("input_299_strides_0"), val = tensor([1])]; - tensor input_299_pad_0 = const()[name = tensor("input_299_pad_0"), val = tensor([0, 0])]; - tensor input_299_dilations_0 = const()[name = tensor("input_299_dilations_0"), val = tensor([1])]; - tensor input_299_groups_0 = const()[name = tensor("input_299_groups_0"), val = tensor(1)]; - tensor input_299 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_299_dilations_0, groups = input_299_groups_0, pad = input_299_pad_0, pad_type = input_299_pad_type_0, strides = input_299_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_297)[name = tensor("input_299")]; - tensor conv_out_37_perm_0 = const()[name = tensor("conv_out_37_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2208_begin_0 = const()[name = tensor("op_2208_begin_0"), val = tensor([0, -1, 0])]; - tensor var_2208_end_0 = const()[name = tensor("op_2208_end_0"), val = tensor([1, 16, 256])]; - tensor var_2208_end_mask_0 = const()[name = tensor("op_2208_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_37 = transpose(perm = conv_out_37_perm_0, x = input_299)[name = tensor("transpose_33")]; - tensor var_2208 = slice_by_index(begin = var_2208_begin_0, end = var_2208_end_0, end_mask = var_2208_end_mask_0, x = conv_out_37)[name = tensor("op_2208")]; - tensor var_2211_begin_0 = const()[name = tensor("op_2211_begin_0"), val = tensor([0, 4, 0])]; - tensor var_2211_end_0 = const()[name = tensor("op_2211_end_0"), val = tensor([1, 1, 256])]; - tensor var_2211_end_mask_0 = const()[name = tensor("op_2211_end_mask_0"), val = tensor([true, true, true])]; - tensor var_2211 = slice_by_index(begin = var_2211_begin_0, end = var_2211_end_0, end_mask = var_2211_end_mask_0, x = x_45)[name = tensor("op_2211")]; - tensor var_2214_begin_0 = const()[name = tensor("op_2214_begin_0"), val = tensor([0, 1, 0])]; - tensor var_2214_end_0 = const()[name = tensor("op_2214_end_0"), val = tensor([1, 16, 256])]; - tensor var_2214_end_mask_0 = const()[name = tensor("op_2214_end_mask_0"), val = tensor([true, true, true])]; - tensor var_2214 = slice_by_index(begin = var_2214_begin_0, end = var_2214_end_0, end_mask = var_2214_end_mask_0, x = input_291)[name = tensor("op_2214")]; + tensor x_expanded_37 = tile(reps = x_expanded_37_reps_0, x = var_1320)[name = tensor("x_expanded_37")]; + tensor var_1334 = sub(x = var_18, y = fill_mask_37)[name = tensor("op_1334")]; + tensor var_1335 = mul(x = window_43, y = var_1334)[name = tensor("op_1335")]; + tensor var_1336 = mul(x = x_expanded_37, y = fill_mask_37)[name = tensor("op_1336")]; + tensor filled_window_37 = add(x = var_1335, y = var_1336)[name = tensor("filled_window_37")]; + tensor var_1338 = greater_equal(x = clip_35, y = var_17)[name = tensor("op_1338")]; + tensor var_1339_dtype_0 = const()[name = tensor("op_1339_dtype_0"), val = tensor("fp32")]; + tensor var_1340 = const()[name = tensor("op_1340"), val = tensor([1, 1, 1])]; + tensor var_1339 = cast(dtype = var_1339_dtype_0, x = var_1338)[name = tensor("cast_154")]; + tensor use_roll_37 = reshape(shape = var_1340, x = var_1339)[name = tensor("use_roll_37")]; + tensor var_1342 = sub(x = var_18, y = use_roll_37)[name = tensor("op_1342")]; + tensor var_1343 = mul(x = filled_window_37, y = var_1342)[name = tensor("op_1343")]; + tensor var_1344 = mul(x = rolled_37, y = use_roll_37)[name = tensor("op_1344")]; + tensor window_45 = add(x = var_1343, y = var_1344)[name = tensor("window_45")]; + tensor var_1346 = const()[name = tensor("op_1346"), val = tensor(0x1p+0)]; + tensor var_1347 = add(x = clip_35, y = var_1346)[name = tensor("op_1347")]; + tensor clip_37 = clip(alpha = var_22, beta = var_17, x = var_1347)[name = tensor("clip_37")]; + tensor var_1350_begin_0 = const()[name = tensor("op_1350_begin_0"), val = tensor([0, 4, 0])]; + tensor var_1350_end_0 = const()[name = tensor("op_1350_end_0"), val = tensor([1, 1, 256])]; + tensor var_1350_end_mask_0 = const()[name = tensor("op_1350_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1350 = slice_by_index(begin = var_1350_begin_0, end = var_1350_end_0, end_mask = var_1350_end_mask_0, x = x_21)[name = tensor("op_1350")]; + tensor var_1353_begin_0 = const()[name = tensor("op_1353_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1353_end_0 = const()[name = tensor("op_1353_end_0"), val = tensor([1, 16, 256])]; + tensor var_1353_end_mask_0 = const()[name = tensor("op_1353_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1353 = slice_by_index(begin = var_1353_begin_0, end = var_1353_end_0, end_mask = var_1353_end_mask_0, x = window_45)[name = tensor("op_1353")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_34, interleave = rolled_interleave_0, values = (var_2214, var_2211))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_34, interleave = rolled_interleave_0, values = (var_1353, var_1350))[name = tensor("rolled")]; tensor clip_38 = clip(alpha = var_22, beta = var_21, x = clip_37)[name = tensor("clip_38")]; - tensor var_2219 = equal(x = encoder__conv_positions, y = clip_38)[name = tensor("op_2219")]; - tensor var_2220_dtype_0 = const()[name = tensor("op_2220_dtype_0"), val = tensor("fp32")]; - tensor var_2221 = const()[name = tensor("op_2221"), val = tensor([1, 16, 1])]; - tensor var_2220 = cast(dtype = var_2220_dtype_0, x = var_2219)[name = tensor("cast_153")]; - tensor fill_mask = reshape(shape = var_2221, x = var_2220)[name = tensor("fill_mask")]; + tensor var_1358 = equal(x = encoder__conv_positions, y = clip_38)[name = tensor("op_1358")]; + tensor var_1359_dtype_0 = const()[name = tensor("op_1359_dtype_0"), val = tensor("fp32")]; + tensor var_1360 = const()[name = tensor("op_1360"), val = tensor([1, 16, 1])]; + tensor var_1359 = cast(dtype = var_1359_dtype_0, x = var_1358)[name = tensor("cast_153")]; + tensor fill_mask = reshape(shape = var_1360, x = var_1359)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_2211)[name = tensor("x_expanded")]; - tensor var_2225 = sub(x = var_18, y = fill_mask)[name = tensor("op_2225")]; - tensor var_2226 = mul(x = input_291, y = var_2225)[name = tensor("op_2226")]; - tensor var_2227 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_2227")]; - tensor filled_window = add(x = var_2226, y = var_2227)[name = tensor("filled_window")]; - tensor var_2229 = greater_equal(x = clip_37, y = var_17)[name = tensor("op_2229")]; - tensor var_2230_dtype_0 = const()[name = tensor("op_2230_dtype_0"), val = tensor("fp32")]; - tensor var_2231 = const()[name = tensor("op_2231"), val = tensor([1, 1, 1])]; - tensor var_2230 = cast(dtype = var_2230_dtype_0, x = var_2229)[name = tensor("cast_152")]; - tensor use_roll = reshape(shape = var_2231, x = var_2230)[name = tensor("use_roll")]; - tensor var_2233 = sub(x = var_18, y = use_roll)[name = tensor("op_2233")]; - tensor var_2234 = mul(x = filled_window, y = var_2233)[name = tensor("op_2234")]; - tensor var_2235 = mul(x = rolled, y = use_roll)[name = tensor("op_2235")]; - tensor input_301 = add(x = var_2234, y = var_2235)[name = tensor("input_301")]; - tensor var_2237 = const()[name = tensor("op_2237"), val = tensor(0x1p+0)]; - tensor var_2238 = add(x = clip_37, y = var_2237)[name = tensor("op_2238")]; - tensor clip_39 = clip(alpha = var_22, beta = var_17, x = var_2238)[name = tensor("clip_39")]; - tensor x_55_axes_0 = const()[name = tensor("x_55_axes_0"), val = tensor([-1])]; - tensor x_55 = layer_norm(axes = x_55_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_301)[name = tensor("x_55")]; - tensor input_303_perm_0 = const()[name = tensor("input_303_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_129_pad_type_0 = const()[name = tensor("inputs_129_pad_type_0"), val = tensor("valid")]; - tensor inputs_129_strides_0 = const()[name = tensor("inputs_129_strides_0"), val = tensor([1])]; - tensor inputs_129_pad_0 = const()[name = tensor("inputs_129_pad_0"), val = tensor([0, 0])]; - tensor inputs_129_dilations_0 = const()[name = tensor("inputs_129_dilations_0"), val = tensor([1])]; - tensor inputs_129_groups_0 = const()[name = tensor("inputs_129_groups_0"), val = tensor(1)]; - tensor input_303 = transpose(perm = input_303_perm_0, x = x_55)[name = tensor("transpose_32")]; - tensor inputs_129 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_129_dilations_0, groups = inputs_129_groups_0, pad = inputs_129_pad_0, pad_type = inputs_129_pad_type_0, strides = inputs_129_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_303)[name = tensor("inputs_129")]; - tensor var_2259_split_sizes_0 = const()[name = tensor("op_2259_split_sizes_0"), val = tensor([256, 256])]; - tensor var_2259_axis_0 = const()[name = tensor("op_2259_axis_0"), val = tensor(1)]; - tensor var_2259_0, tensor var_2259_1 = split(axis = var_2259_axis_0, split_sizes = var_2259_split_sizes_0, x = inputs_129)[name = tensor("op_2259")]; - tensor var_2261 = sigmoid(x = var_2259_1)[name = tensor("op_2261")]; - tensor inputs_131 = mul(x = var_2259_0, y = var_2261)[name = tensor("inputs_131")]; + tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1350)[name = tensor("x_expanded")]; + tensor var_1364 = sub(x = var_18, y = fill_mask)[name = tensor("op_1364")]; + tensor var_1365 = mul(x = window_45, y = var_1364)[name = tensor("op_1365")]; + tensor var_1366 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1366")]; + tensor filled_window = add(x = var_1365, y = var_1366)[name = tensor("filled_window")]; + tensor var_1368 = greater_equal(x = clip_37, y = var_17)[name = tensor("op_1368")]; + tensor var_1369_dtype_0 = const()[name = tensor("op_1369_dtype_0"), val = tensor("fp32")]; + tensor var_1370 = const()[name = tensor("op_1370"), val = tensor([1, 1, 1])]; + tensor var_1369 = cast(dtype = var_1369_dtype_0, x = var_1368)[name = tensor("cast_152")]; + tensor use_roll = reshape(shape = var_1370, x = var_1369)[name = tensor("use_roll")]; + tensor var_1372 = sub(x = var_18, y = use_roll)[name = tensor("op_1372")]; + tensor var_1373 = mul(x = filled_window, y = var_1372)[name = tensor("op_1373")]; + tensor var_1374 = mul(x = rolled, y = use_roll)[name = tensor("op_1374")]; + tensor window = add(x = var_1373, y = var_1374)[name = tensor("window")]; + tensor var_1376 = const()[name = tensor("op_1376"), val = tensor(0x1p+0)]; + tensor var_1377 = add(x = clip_37, y = var_1376)[name = tensor("op_1377")]; + tensor clip_39 = clip(alpha = var_22, beta = var_17, x = var_1377)[name = tensor("clip_39")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_31, interleave = input_141_interleave_0, values = (window_39, window_41, window_43, window_45, window))[name = tensor("input_141")]; + tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; + tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; + tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_33_pad_type_0 = const()[name = tensor("inputs_33_pad_type_0"), val = tensor("valid")]; + tensor inputs_33_strides_0 = const()[name = tensor("inputs_33_strides_0"), val = tensor([1])]; + tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; + tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; + tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; + tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; + tensor var_1400_split_sizes_0 = const()[name = tensor("op_1400_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1400_axis_0 = const()[name = tensor("op_1400_axis_0"), val = tensor(1)]; + tensor var_1400_0, tensor var_1400_1 = split(axis = var_1400_axis_0, split_sizes = var_1400_split_sizes_0, x = inputs_33)[name = tensor("op_1400")]; + tensor var_1402 = sigmoid(x = var_1400_1)[name = tensor("op_1402")]; + tensor inputs_35 = mul(x = var_1400_0, y = var_1402)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; tensor outputs_aug_strides_0 = const()[name = tensor("outputs_aug_strides_0"), val = tensor([1])]; tensor outputs_aug_dilations_0 = const()[name = tensor("outputs_aug_dilations_0"), val = tensor([1])]; - tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_131)[name = tensor("outputs_aug")]; - tensor input_305_begin_0 = const()[name = tensor("input_305_begin_0"), val = tensor([0, 0, 0])]; - tensor input_305_end_0 = const()[name = tensor("input_305_end_0"), val = tensor([1, 256, 16])]; - tensor input_305_end_mask_0 = const()[name = tensor("input_305_end_mask_0"), val = tensor([true, true, false])]; - tensor input_305 = slice_by_index(begin = input_305_begin_0, end = input_305_end_0, end_mask = input_305_end_mask_0, x = outputs_aug)[name = tensor("input_305")]; - tensor inputs_133 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_305)[name = tensor("inputs_133")]; - tensor input_307 = silu(x = inputs_133)[name = tensor("input_307")]; - tensor input_309_pad_type_0 = const()[name = tensor("input_309_pad_type_0"), val = tensor("valid")]; - tensor input_309_strides_0 = const()[name = tensor("input_309_strides_0"), val = tensor([1])]; - tensor input_309_pad_0 = const()[name = tensor("input_309_pad_0"), val = tensor([0, 0])]; - tensor input_309_dilations_0 = const()[name = tensor("input_309_dilations_0"), val = tensor([1])]; - tensor input_309_groups_0 = const()[name = tensor("input_309_groups_0"), val = tensor(1)]; - tensor input_309 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_309_dilations_0, groups = input_309_groups_0, pad = input_309_pad_0, pad_type = input_309_pad_type_0, strides = input_309_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_307)[name = tensor("input_309")]; - tensor conv_out_39_perm_0 = const()[name = tensor("conv_out_39_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2292_begin_0 = const()[name = tensor("op_2292_begin_0"), val = tensor([0, -1, 0])]; - tensor var_2292_end_0 = const()[name = tensor("op_2292_end_0"), val = tensor([1, 16, 256])]; - tensor var_2292_end_mask_0 = const()[name = tensor("op_2292_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_39 = transpose(perm = conv_out_39_perm_0, x = input_309)[name = tensor("transpose_31")]; - tensor var_2292 = slice_by_index(begin = var_2292_begin_0, end = var_2292_end_0, end_mask = var_2292_end_mask_0, x = conv_out_39)[name = tensor("op_2292")]; - tensor var_2295_interleave_0 = const()[name = tensor("op_2295_interleave_0"), val = tensor(false)]; - tensor var_2295 = concat(axis = var_34, interleave = var_2295_interleave_0, values = (var_1956, var_2040, var_2124, var_2208, var_2292))[name = tensor("op_2295")]; - tensor input_311 = add(x = x_45, y = var_2295)[name = tensor("input_311")]; - tensor input_313_axes_0 = const()[name = tensor("input_313_axes_0"), val = tensor([-1])]; - tensor input_313 = layer_norm(axes = input_313_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_311)[name = tensor("input_313")]; - tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_313)[name = tensor("linear_35")]; - tensor input_315 = silu(x = inputs)[name = tensor("input_315")]; - tensor input_319 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_315)[name = tensor("linear_36")]; - tensor var_2318 = const()[name = tensor("op_2318"), val = tensor(0x1p-1)]; - tensor var_2319 = mul(x = input_319, y = var_2318)[name = tensor("op_2319")]; - tensor input_321 = add(x = var_2319, y = input_311)[name = tensor("input_321")]; - tensor x_57_axes_0 = const()[name = tensor("x_57_axes_0"), val = tensor([-1])]; - tensor x_57 = layer_norm(axes = x_57_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_36, gamma = encoder_layer_norm_3_weight, x = input_321)[name = tensor("x_57")]; + tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_35)[name = tensor("outputs_aug")]; + tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; + tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([5, 256, 16])]; + tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; + tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug)[name = tensor("input_145")]; + tensor inputs_37 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_145)[name = tensor("inputs_37")]; + tensor input_147 = silu(x = inputs_37)[name = tensor("input_147")]; + tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; + tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; + tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; + tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; + tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; + tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; + tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1433_begin_0 = const()[name = tensor("op_1433_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1433_end_0 = const()[name = tensor("op_1433_end_0"), val = tensor([5, 16, 256])]; + tensor var_1433_end_mask_0 = const()[name = tensor("op_1433_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_1433 = slice_by_index(begin = var_1433_begin_0, end = var_1433_end_0, end_mask = var_1433_end_mask_0, x = conv_out_7)[name = tensor("op_1433")]; + tensor var_1435_perm_0 = const()[name = tensor("op_1435_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1435 = transpose(perm = var_1435_perm_0, x = var_1433)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_1435)[name = tensor("input_151")]; + tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; + tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; + tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; + tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; + tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; + tensor var_1458 = const()[name = tensor("op_1458"), val = tensor(0x1p-1)]; + tensor var_1459 = mul(x = input_159, y = var_1458)[name = tensor("op_1459")]; + tensor input_161 = add(x = var_1459, y = input_151)[name = tensor("input_161")]; + tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; + tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_36, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; tensor cat_interleave_0 = const()[name = tensor("cat_interleave_0"), val = tensor(false)]; - tensor x_ct = transpose(perm = x_ct_perm_0, x = x_57)[name = tensor("transpose_30")]; + tensor x_ct = transpose(perm = x_ct_perm_0, x = x_25)[name = tensor("transpose_30")]; tensor cat = concat(axis = var_28, interleave = cat_interleave_0, values = (cnn_window, x_ct))[name = tensor("cat")]; tensor conv_out_pad_type_0 = const()[name = tensor("conv_out_pad_type_0"), val = tensor("valid")]; tensor conv_out_strides_0 = const()[name = tensor("conv_out_strides_0"), val = tensor([1])]; @@ -2097,44 +1482,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_2337_begin_0 = const()[name = tensor("op_2337_begin_0"), val = tensor([0, 0, 5])]; - tensor var_2337_end_0 = const()[name = tensor("op_2337_end_0"), val = tensor([1, 256, 23])]; - tensor var_2337_end_mask_0 = const()[name = tensor("op_2337_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_2337_begin_0, end = var_2337_end_0, end_mask = var_2337_end_mask_0, x = cat)[name = tensor("op_2337")]; - tensor input_323_perm_0 = const()[name = tensor("input_323_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2339 = const()[name = tensor("op_2339"), val = tensor([-1])]; - tensor input_323 = transpose(perm = input_323_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_2340 = reduce_l2_norm(axes = var_2339, keep_dims = var_37, x = input_323)[name = tensor("op_2340")]; - tensor const_28 = const()[name = tensor("const_28"), val = tensor(0x1.fffffep+127)]; - tensor clip_40 = clip(alpha = var_12, beta = const_28, x = var_2340)[name = tensor("clip_40")]; - tensor emb = real_div(x = input_323, y = clip_40)[name = tensor("emb")]; - tensor var_2344_axis_0 = const()[name = tensor("op_2344_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_2344_axis_0, values = (var_215, var_763, var_1311, nkv_1))[name = tensor("op_2344")]; - tensor var_2346_axis_0 = const()[name = tensor("op_2346_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_2346_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_2346")]; - tensor var_2348_axis_0 = const()[name = tensor("op_2348_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_2348_axis_0, values = (input_61, input_141, input_221, input_301))[name = tensor("op_2348")]; - tensor var_2350_axis_0 = const()[name = tensor("op_2350_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_2350_axis_0, values = (clip_9, clip_19, clip_29, clip_39))[name = tensor("op_2350")]; - tensor var_2360 = const()[name = tensor("op_2360"), val = tensor(0x1.5798eep-27)]; - tensor var_2366 = const()[name = tensor("op_2366"), val = tensor(0x1.4f8b58p-17)]; - tensor var_2368 = const()[name = tensor("op_2368"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_2369 = const()[name = tensor("op_2369"), val = tensor(true)]; - tensor var_2371 = const()[name = tensor("op_2371"), val = tensor(0x1p+0)]; - tensor var_2375 = const()[name = tensor("op_2375"), val = tensor(-1)]; - tensor var_2380 = const()[name = tensor("op_2380"), val = tensor(0)]; + tensor var_1477_begin_0 = const()[name = tensor("op_1477_begin_0"), val = tensor([0, 0, 5])]; + tensor var_1477_end_0 = const()[name = tensor("op_1477_end_0"), val = tensor([1, 256, 23])]; + tensor var_1477_end_mask_0 = const()[name = tensor("op_1477_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_1477_begin_0, end = var_1477_end_0, end_mask = var_1477_end_mask_0, x = cat)[name = tensor("op_1477")]; + tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1479 = const()[name = tensor("op_1479"), val = tensor([-1])]; + tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; + tensor var_1480 = reduce_l2_norm(axes = var_1479, keep_dims = var_37, x = input_163)[name = tensor("op_1480")]; + tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; + tensor clip_40 = clip(alpha = var_12, beta = const_12, x = var_1480)[name = tensor("clip_40")]; + tensor emb = real_div(x = input_163, y = clip_40)[name = tensor("emb")]; + tensor var_1484_axis_0 = const()[name = tensor("op_1484_axis_0"), val = tensor(0)]; + tensor enc_kv_new = stack(axis = var_1484_axis_0, values = (var_215, var_548, var_881, nkv_1))[name = tensor("op_1484")]; + tensor var_1486_axis_0 = const()[name = tensor("op_1486_axis_0"), val = tensor(0)]; + tensor enc_scale_new = stack(axis = var_1486_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1486")]; + tensor var_1488_axis_0 = const()[name = tensor("op_1488_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1488_axis_0, values = (window_11, window_23, window_35, window))[name = tensor("op_1488")]; + tensor var_1490_axis_0 = const()[name = tensor("op_1490_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1490_axis_0, values = (clip_9, clip_19, clip_29, clip_39))[name = tensor("op_1490")]; + tensor var_1500 = const()[name = tensor("op_1500"), val = tensor(0x1.5798eep-27)]; + tensor var_1505 = const()[name = tensor("op_1505"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1507 = const()[name = tensor("op_1507"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1508 = const()[name = tensor("op_1508"), val = tensor(true)]; + tensor var_1510 = const()[name = tensor("op_1510"), val = tensor(0x1p+0)]; + tensor var_1514 = const()[name = tensor("op_1514"), val = tensor(-1)]; + tensor var_1520 = const()[name = tensor("op_1520"), val = tensor(0)]; tensor pos = const()[name = tensor("pos"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395840)))]; - tensor var_2442_axes_0 = const()[name = tensor("op_2442_axes_0"), val = tensor([2])]; - tensor var_2442 = expand_dims(axes = var_2442_axes_0, x = emb)[name = tensor("op_2442")]; - tensor var_2443 = const()[name = tensor("op_2443"), val = tensor([1, 1, 9, 1])]; - tensor emb_exp = tile(reps = var_2443, x = var_2442)[name = tensor("emb_exp")]; - tensor input_325_interleave_0 = const()[name = tensor("input_325_interleave_0"), val = tensor(false)]; - tensor input_325 = concat(axis = var_2375, interleave = input_325_interleave_0, values = (emb_exp, pos))[name = tensor("input_325")]; - tensor x_59 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_325)[name = tensor("linear_37")]; - tensor var_2450_perm_0 = const()[name = tensor("op_2450_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2454 = const()[name = tensor("op_2454"), val = tensor([9, 5, 256])]; - tensor var_2450 = transpose(perm = var_2450_perm_0, x = x_59)[name = tensor("transpose_28")]; - tensor x_61 = reshape(shape = var_2454, x = var_2450)[name = tensor("x_61")]; + tensor var_1582_axes_0 = const()[name = tensor("op_1582_axes_0"), val = tensor([2])]; + tensor var_1582 = expand_dims(axes = var_1582_axes_0, x = emb)[name = tensor("op_1582")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 9, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1582)[name = tensor("emb_exp")]; + tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; + tensor input_165 = concat(axis = var_1514, interleave = input_165_interleave_0, values = (emb_exp, pos))[name = tensor("input_165")]; + tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; + tensor var_1590_perm_0 = const()[name = tensor("op_1590_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1594 = const()[name = tensor("op_1594"), val = tensor([9, 5, 256])]; + tensor var_1590 = transpose(perm = var_1590_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1594, x = var_1590)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 9, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -2145,132 +1530,132 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_2462 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_61)[name = tensor("linear_38")]; - tensor var_2463 = const()[name = tensor("op_2463"), val = tensor([9, 5, 4, 64])]; - tensor var_2464 = reshape(shape = var_2463, x = var_2462)[name = tensor("op_2464")]; + tensor var_1602 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1603 = const()[name = tensor("op_1603"), val = tensor([9, 5, 4, 64])]; + tensor var_1604 = reshape(shape = var_1603, x = var_1602)[name = tensor("op_1604")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2468 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_61)[name = tensor("linear_39")]; - tensor var_2469 = const()[name = tensor("op_2469"), val = tensor(0x1p-3)]; - tensor var_2470 = mul(x = var_2468, y = var_2469)[name = tensor("op_2470")]; - tensor var_2471 = const()[name = tensor("op_2471"), val = tensor([9, 5, 4, 64])]; - tensor var_2472 = reshape(shape = var_2471, x = var_2470)[name = tensor("op_2472")]; + tensor var_1608 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1609 = const()[name = tensor("op_1609"), val = tensor(0x1p-3)]; + tensor var_1610 = mul(x = var_1608, y = var_1609)[name = tensor("op_1610")]; + tensor var_1611 = const()[name = tensor("op_1611"), val = tensor([9, 5, 4, 64])]; + tensor var_1612 = reshape(shape = var_1611, x = var_1610)[name = tensor("op_1612")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2476 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_61)[name = tensor("linear_40")]; - tensor var_2477 = const()[name = tensor("op_2477"), val = tensor([9, 5, 4, 64])]; - tensor var_2478 = reshape(shape = var_2477, x = var_2476)[name = tensor("op_2478")]; + tensor var_1616 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1617 = const()[name = tensor("op_1617"), val = tensor([9, 5, 4, 64])]; + tensor var_1618 = reshape(shape = var_1617, x = var_1616)[name = tensor("op_1618")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_329 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_61)[name = tensor("linear_41")]; + tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_2380, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1520, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; - tensor const_36 = const()[name = tensor("const_36"), val = tensor(0x1.fffffep+127)]; - tensor clip_41 = clip(alpha = var_2371, beta = const_36, x = s_t_9)[name = tensor("clip_41")]; + tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; + tensor clip_41 = clip(alpha = var_1510, beta = const_20, x = s_t_9)[name = tensor("clip_41")]; tensor sqrt_s_t_9 = sqrt(x = clip_41)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_2472)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_2464)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1612)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1604)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_2490 = const()[name = tensor("op_2490"), val = tensor([1, 5])]; - tensor var_2491 = reshape(shape = var_2490, x = valid_mask)[name = tensor("op_2491")]; - tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_2491)[name = tensor("causal_with_valid_1")]; - tensor var_2493 = const()[name = tensor("op_2493"), val = tensor([5, 1])]; - tensor var_2494 = reshape(shape = var_2493, x = sqrt_s_t_9)[name = tensor("op_2494")]; - tensor M_9 = real_div(x = causal_with_valid_1, y = var_2494)[name = tensor("M_9")]; - tensor var_2496 = mul(x = qk_9, y = M_9)[name = tensor("op_2496")]; + tensor var_1630 = const()[name = tensor("op_1630"), val = tensor([1, 5])]; + tensor var_1631 = reshape(shape = var_1630, x = valid_mask)[name = tensor("op_1631")]; + tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1631)[name = tensor("causal_with_valid_1")]; + tensor var_1633 = const()[name = tensor("op_1633"), val = tensor([5, 1])]; + tensor var_1634 = reshape(shape = var_1633, x = sqrt_s_t_9)[name = tensor("op_1634")]; + tensor M_9 = real_div(x = causal_with_valid_1, y = var_1634)[name = tensor("M_9")]; + tensor var_1636 = mul(x = qk_9, y = M_9)[name = tensor("op_1636")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_2478)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_2496, y = v_9)[name = tensor("inner_9")]; - tensor var_2498_transpose_x_0 = const()[name = tensor("op_2498_transpose_x_0"), val = tensor(false)]; - tensor var_2498_transpose_y_0 = const()[name = tensor("op_2498_transpose_y_0"), val = tensor(false)]; - tensor var_2498 = matmul(transpose_x = var_2498_transpose_x_0, transpose_y = var_2498_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_2498")]; - tensor var_2499 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_2499")]; - tensor var_2500 = const()[name = tensor("op_2500"), val = tensor([1, 1, 5, 1])]; - tensor var_2501 = reshape(shape = var_2500, x = var_2499)[name = tensor("op_2501")]; - tensor cross_9 = mul(x = var_2498, y = var_2501)[name = tensor("cross_9")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1618)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1636, y = v_9)[name = tensor("inner_9")]; + tensor var_1638_transpose_x_0 = const()[name = tensor("op_1638_transpose_x_0"), val = tensor(false)]; + tensor var_1638_transpose_y_0 = const()[name = tensor("op_1638_transpose_y_0"), val = tensor(false)]; + tensor var_1638 = matmul(transpose_x = var_1638_transpose_x_0, transpose_y = var_1638_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1638")]; + tensor var_1639 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1639")]; + tensor var_1640 = const()[name = tensor("op_1640"), val = tensor([1, 1, 5, 1])]; + tensor var_1641 = reshape(shape = var_1640, x = var_1639)[name = tensor("op_1641")]; + tensor cross_9 = mul(x = var_1638, y = var_1641)[name = tensor("cross_9")]; tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; - tensor var_2504 = const()[name = tensor("op_2504"), val = tensor([1, 1, 5, 1])]; - tensor var_2505 = reshape(shape = var_2504, x = valid_mask)[name = tensor("op_2505")]; - tensor v_masked_1 = mul(x = v_9, y = var_2505)[name = tensor("v_masked_1")]; - tensor var_2507 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_2507")]; - tensor var_2509_transpose_x_1 = const()[name = tensor("op_2509_transpose_x_1"), val = tensor(true)]; - tensor var_2509_transpose_y_1 = const()[name = tensor("op_2509_transpose_y_1"), val = tensor(false)]; - tensor var_2509 = matmul(transpose_x = var_2509_transpose_x_1, transpose_y = var_2509_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_2509")]; - tensor new_kv_unnorm_9 = add(x = var_2507, y = var_2509)[name = tensor("new_kv_unnorm_9")]; - tensor var_2511_keep_dims_0 = const()[name = tensor("op_2511_keep_dims_0"), val = tensor(false)]; - tensor var_2511 = reduce_sum(keep_dims = var_2511_keep_dims_0, x = valid_mask)[name = tensor("op_2511")]; - tensor var_2512 = const()[name = tensor("op_2512"), val = tensor([1])]; - tensor var_2513 = reshape(shape = var_2512, x = var_2511)[name = tensor("op_2513")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_2513)[name = tensor("new_scale_9")]; - tensor const_37 = const()[name = tensor("const_37"), val = tensor(0x1.fffffep+127)]; - tensor clip_42 = clip(alpha = var_2371, beta = const_37, x = new_scale_9)[name = tensor("clip_42")]; + tensor var_1644 = const()[name = tensor("op_1644"), val = tensor([1, 1, 5, 1])]; + tensor var_1645 = reshape(shape = var_1644, x = valid_mask)[name = tensor("op_1645")]; + tensor v_masked_1 = mul(x = v_9, y = var_1645)[name = tensor("v_masked_1")]; + tensor var_1647 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1647")]; + tensor var_1649_transpose_x_1 = const()[name = tensor("op_1649_transpose_x_1"), val = tensor(true)]; + tensor var_1649_transpose_y_1 = const()[name = tensor("op_1649_transpose_y_1"), val = tensor(false)]; + tensor var_1649 = matmul(transpose_x = var_1649_transpose_x_1, transpose_y = var_1649_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1649")]; + tensor new_kv_unnorm_9 = add(x = var_1647, y = var_1649)[name = tensor("new_kv_unnorm_9")]; + tensor var_1651_keep_dims_0 = const()[name = tensor("op_1651_keep_dims_0"), val = tensor(false)]; + tensor var_1651 = reduce_sum(keep_dims = var_1651_keep_dims_0, x = valid_mask)[name = tensor("op_1651")]; + tensor var_1652 = const()[name = tensor("op_1652"), val = tensor([1])]; + tensor var_1653 = reshape(shape = var_1652, x = var_1651)[name = tensor("op_1653")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1653)[name = tensor("new_scale_9")]; + tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; + tensor clip_42 = clip(alpha = var_1510, beta = const_21, x = new_scale_9)[name = tensor("clip_42")]; tensor sqrt_new_scale_1 = sqrt(x = clip_42)[name = tensor("sqrt_new_scale_1")]; - tensor var_2517 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_2517")]; - tensor var_2518_perm_0 = const()[name = tensor("op_2518_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1657 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1657")]; + tensor var_1658_perm_0 = const()[name = tensor("op_1658_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_2518 = transpose(perm = var_2518_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_2368, x = var_2518)[name = tensor("out_27")]; - tensor var_2522 = const()[name = tensor("op_2522"), val = tensor([9, 5, 256])]; - tensor out_29 = reshape(shape = var_2522, x = out_27)[name = tensor("out_29")]; - tensor var_2524 = silu(x = input_329)[name = tensor("op_2524")]; - tensor input_331 = mul(x = var_2524, y = out_29)[name = tensor("input_331")]; - tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_331)[name = tensor("linear_42")]; - tensor input_333 = add(x = x_61, y = ret_out_9)[name = tensor("input_333")]; + tensor var_1658 = transpose(perm = var_1658_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1507, x = var_1658)[name = tensor("out_27")]; + tensor var_1662 = const()[name = tensor("op_1662"), val = tensor([9, 5, 256])]; + tensor out_29 = reshape(shape = var_1662, x = out_27)[name = tensor("out_29")]; + tensor var_1664 = silu(x = input_169)[name = tensor("op_1664")]; + tensor input_171 = mul(x = var_1664, y = out_29)[name = tensor("input_171")]; + tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; + tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_2366, gamma = decoder_norm11_0_weight, x = input_333)[name = tensor("xt_1")]; - tensor var_2534 = const()[name = tensor("op_2534"), val = tensor([1, 9, 5, 256])]; - tensor var_2535 = reshape(shape = var_2534, x = xt_1)[name = tensor("op_2535")]; - tensor var_2536_perm_0 = const()[name = tensor("op_2536_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2539 = const()[name = tensor("op_2539"), val = tensor([5, 9, 256])]; - tensor var_2536 = transpose(perm = var_2536_perm_0, x = var_2535)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_2539, x = var_2536)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1505, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1674 = const()[name = tensor("op_1674"), val = tensor([1, 9, 5, 256])]; + tensor var_1675 = reshape(shape = var_1674, x = xt_1)[name = tensor("op_1675")]; + tensor var_1676_perm_0 = const()[name = tensor("op_1676_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1679 = const()[name = tensor("op_1679"), val = tensor([5, 9, 256])]; + tensor var_1676 = transpose(perm = var_1676_perm_0, x = var_1675)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1679, x = var_1676)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_2562 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_20 = const()[name = tensor("concat_20"), val = tensor([9, 5, 3, 256])]; - tensor var_2564 = reshape(shape = concat_20, x = var_2562)[name = tensor("op_2564")]; - tensor var_2565_axes_0 = const()[name = tensor("op_2565_axes_0"), val = tensor([0])]; - tensor var_2565 = expand_dims(axes = var_2565_axes_0, x = var_2564)[name = tensor("op_2565")]; - tensor var_2566_perm_0 = const()[name = tensor("op_2566_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2567_axes_0 = const()[name = tensor("op_2567_axes_0"), val = tensor([-2])]; - tensor var_2566 = transpose(perm = var_2566_perm_0, x = var_2565)[name = tensor("transpose_21")]; - tensor var_2567 = squeeze(axes = var_2567_axes_0, x = var_2566)[name = tensor("op_2567")]; + tensor var_1702 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_21 = const()[name = tensor("concat_21"), val = tensor([9, 5, 3, 256])]; + tensor var_1704 = reshape(shape = concat_21, x = var_1702)[name = tensor("op_1704")]; + tensor var_1705_axes_0 = const()[name = tensor("op_1705_axes_0"), val = tensor([0])]; + tensor var_1705 = expand_dims(axes = var_1705_axes_0, x = var_1704)[name = tensor("op_1705")]; + tensor var_1706_perm_0 = const()[name = tensor("op_1706_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1707_axes_0 = const()[name = tensor("op_1707_axes_0"), val = tensor([-2])]; + tensor var_1706 = transpose(perm = var_1706_perm_0, x = var_1705)[name = tensor("transpose_21")]; + tensor var_1707 = squeeze(axes = var_1707_axes_0, x = var_1706)[name = tensor("op_1707")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 9, 5, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_2567)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1707)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 9, 5, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_2567)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1707)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 9, 5, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_2567)[name = tensor("v_11")]; - tensor var_2575 = const()[name = tensor("op_2575"), val = tensor([9, 20, 64])]; - tensor var_2576 = reshape(shape = var_2575, x = q_11)[name = tensor("op_2576")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1707)[name = tensor("v_11")]; + tensor var_1715 = const()[name = tensor("op_1715"), val = tensor([9, 20, 64])]; + tensor var_1716 = reshape(shape = var_1715, x = q_11)[name = tensor("op_1716")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2582 = const()[name = tensor("op_2582"), val = tensor([9, 20, 64])]; - tensor var_2583 = reshape(shape = var_2582, x = k_11)[name = tensor("op_2583")]; + tensor var_1722 = const()[name = tensor("op_1722"), val = tensor([9, 20, 64])]; + tensor var_1723 = reshape(shape = var_1722, x = k_11)[name = tensor("op_1723")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2589 = const()[name = tensor("op_2589"), val = tensor([9, 20, 64])]; - tensor var_2590 = reshape(shape = var_2589, x = v_11)[name = tensor("op_2590")]; + tensor var_1729 = const()[name = tensor("op_1729"), val = tensor([9, 20, 64])]; + tensor var_1730 = reshape(shape = var_1729, x = v_11)[name = tensor("op_1730")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2593 = const()[name = tensor("op_2593"), val = tensor([5, 4, 9, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_2576)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_2593, x = q_13)[name = tensor("q_15")]; - tensor var_2595 = const()[name = tensor("op_2595"), val = tensor([5, 4, 9, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_2583)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_2595, x = k_13)[name = tensor("k_15")]; - tensor var_2597 = const()[name = tensor("op_2597"), val = tensor([5, 4, 9, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_2590)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_2597, x = v_13)[name = tensor("v_15")]; + tensor var_1733 = const()[name = tensor("op_1733"), val = tensor([5, 4, 9, 64])]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1716)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1733, x = q_13)[name = tensor("q_15")]; + tensor var_1735 = const()[name = tensor("op_1735"), val = tensor([5, 4, 9, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1723)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1735, x = k_13)[name = tensor("k_15")]; + tensor var_1737 = const()[name = tensor("op_1737"), val = tensor([5, 4, 9, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1730)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1737, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -2281,30 +1666,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_2600 = const()[name = tensor("op_2600"), val = tensor([2, 0, 1, 3])]; - tensor var_2605 = const()[name = tensor("op_2605"), val = tensor([45, 256])]; - tensor var_2601 = transpose(perm = var_2600, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_2605, x = var_2601)[name = tensor("attn_output_3")]; + tensor var_1740 = const()[name = tensor("op_1740"), val = tensor([2, 0, 1, 3])]; + tensor var_1745 = const()[name = tensor("op_1745"), val = tensor([45, 256])]; + tensor var_1741 = transpose(perm = var_1740, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1745, x = var_1741)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_2609 = const()[name = tensor("op_2609"), val = tensor([9, 5, 256])]; - tensor attn_output_7 = reshape(shape = var_2609, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1749 = const()[name = tensor("op_1749"), val = tensor([9, 5, 256])]; + tensor attn_output_7 = reshape(shape = var_1749, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; - tensor input_335 = add(x = query_1, y = sa2_out_1)[name = tensor("input_335")]; - tensor input_337_axes_0 = const()[name = tensor("input_337_axes_0"), val = tensor([-1])]; - tensor input_337 = layer_norm(axes = input_337_axes_0, beta = decoder_norm21_0_bias, epsilon = var_2366, gamma = decoder_norm21_0_weight, x = input_335)[name = tensor("input_337")]; - tensor input_339 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_337)[name = tensor("linear_45")]; - tensor input_341 = relu(x = input_339)[name = tensor("input_341")]; - tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_341)[name = tensor("linear_46")]; - tensor input_343 = add(x = input_337, y = ffn_out_1)[name = tensor("input_343")]; + tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; + tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1505, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; + tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; + tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; + tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_2366, gamma = decoder_norm22_0_weight, x = input_343)[name = tensor("xt_3")]; - tensor var_2629 = const()[name = tensor("op_2629"), val = tensor([1, 5, 9, 256])]; - tensor x_63 = reshape(shape = var_2629, x = xt_3)[name = tensor("x_63")]; - tensor var_2631_perm_0 = const()[name = tensor("op_2631_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2635 = const()[name = tensor("op_2635"), val = tensor([9, 5, 256])]; - tensor var_2631 = transpose(perm = var_2631_perm_0, x = x_63)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_2635, x = var_2631)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1505, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1769 = const()[name = tensor("op_1769"), val = tensor([1, 5, 9, 256])]; + tensor x_31 = reshape(shape = var_1769, x = xt_3)[name = tensor("x_31")]; + tensor var_1771_perm_0 = const()[name = tensor("op_1771_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1775 = const()[name = tensor("op_1775"), val = tensor([9, 5, 256])]; + tensor var_1771 = transpose(perm = var_1771_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1775, x = var_1771)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 9, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -2315,120 +1700,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_2643 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_2644 = const()[name = tensor("op_2644"), val = tensor([9, 5, 4, 64])]; - tensor var_2645 = reshape(shape = var_2644, x = var_2643)[name = tensor("op_2645")]; + tensor var_1783 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1784 = const()[name = tensor("op_1784"), val = tensor([9, 5, 4, 64])]; + tensor var_1785 = reshape(shape = var_1784, x = var_1783)[name = tensor("op_1785")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2649 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_2650 = const()[name = tensor("op_2650"), val = tensor(0x1p-3)]; - tensor var_2651 = mul(x = var_2649, y = var_2650)[name = tensor("op_2651")]; - tensor var_2652 = const()[name = tensor("op_2652"), val = tensor([9, 5, 4, 64])]; - tensor var_2653 = reshape(shape = var_2652, x = var_2651)[name = tensor("op_2653")]; + tensor var_1789 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1790 = const()[name = tensor("op_1790"), val = tensor(0x1p-3)]; + tensor var_1791 = mul(x = var_1789, y = var_1790)[name = tensor("op_1791")]; + tensor var_1792 = const()[name = tensor("op_1792"), val = tensor([9, 5, 4, 64])]; + tensor var_1793 = reshape(shape = var_1792, x = var_1791)[name = tensor("op_1793")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2657 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_2658 = const()[name = tensor("op_2658"), val = tensor([9, 5, 4, 64])]; - tensor var_2659 = reshape(shape = var_2658, x = var_2657)[name = tensor("op_2659")]; + tensor var_1797 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1798 = const()[name = tensor("op_1798"), val = tensor([9, 5, 4, 64])]; + tensor var_1799 = reshape(shape = var_1798, x = var_1797)[name = tensor("op_1799")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_347 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; + tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; - tensor const_48 = const()[name = tensor("const_48"), val = tensor(0x1.fffffep+127)]; - tensor clip_43 = clip(alpha = var_2371, beta = const_48, x = s_t)[name = tensor("clip_43")]; + tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; + tensor clip_43 = clip(alpha = var_1510, beta = const_32, x = s_t)[name = tensor("clip_43")]; tensor sqrt_s_t = sqrt(x = clip_43)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_2653)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_2645)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1793)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1785)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_2674 = const()[name = tensor("op_2674"), val = tensor([5, 1])]; - tensor var_2675 = reshape(shape = var_2674, x = sqrt_s_t)[name = tensor("op_2675")]; - tensor M = real_div(x = causal_with_valid_1, y = var_2675)[name = tensor("M")]; - tensor var_2677 = mul(x = qk, y = M)[name = tensor("op_2677")]; + tensor var_1814 = const()[name = tensor("op_1814"), val = tensor([5, 1])]; + tensor var_1815 = reshape(shape = var_1814, x = sqrt_s_t)[name = tensor("op_1815")]; + tensor M = real_div(x = causal_with_valid_1, y = var_1815)[name = tensor("M")]; + tensor var_1817 = mul(x = qk, y = M)[name = tensor("op_1817")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_2659)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_2677, y = v_17)[name = tensor("inner")]; - tensor var_2679_transpose_x_0 = const()[name = tensor("op_2679_transpose_x_0"), val = tensor(false)]; - tensor var_2679_transpose_y_0 = const()[name = tensor("op_2679_transpose_y_0"), val = tensor(false)]; - tensor var_2679 = matmul(transpose_x = var_2679_transpose_x_0, transpose_y = var_2679_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_2679")]; - tensor var_2680 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_2680")]; - tensor var_2681 = const()[name = tensor("op_2681"), val = tensor([1, 1, 5, 1])]; - tensor var_2682 = reshape(shape = var_2681, x = var_2680)[name = tensor("op_2682")]; - tensor cross = mul(x = var_2679, y = var_2682)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1799)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1817, y = v_17)[name = tensor("inner")]; + tensor var_1819_transpose_x_0 = const()[name = tensor("op_1819_transpose_x_0"), val = tensor(false)]; + tensor var_1819_transpose_y_0 = const()[name = tensor("op_1819_transpose_y_0"), val = tensor(false)]; + tensor var_1819 = matmul(transpose_x = var_1819_transpose_x_0, transpose_y = var_1819_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1819")]; + tensor var_1820 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1820")]; + tensor var_1821 = const()[name = tensor("op_1821"), val = tensor([1, 1, 5, 1])]; + tensor var_1822 = reshape(shape = var_1821, x = var_1820)[name = tensor("op_1822")]; + tensor cross = mul(x = var_1819, y = var_1822)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_2505)[name = tensor("v_masked")]; - tensor var_2688 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_2688")]; - tensor var_2690_transpose_x_1 = const()[name = tensor("op_2690_transpose_x_1"), val = tensor(true)]; - tensor var_2690_transpose_y_1 = const()[name = tensor("op_2690_transpose_y_1"), val = tensor(false)]; - tensor var_2690 = matmul(transpose_x = var_2690_transpose_x_1, transpose_y = var_2690_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_2690")]; - tensor new_kv_unnorm = add(x = var_2688, y = var_2690)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_2513)[name = tensor("new_scale")]; - tensor const_49 = const()[name = tensor("const_49"), val = tensor(0x1.fffffep+127)]; - tensor clip_44 = clip(alpha = var_2371, beta = const_49, x = new_scale)[name = tensor("clip_44")]; + tensor v_masked = mul(x = v_17, y = var_1645)[name = tensor("v_masked")]; + tensor var_1828 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1828")]; + tensor var_1830_transpose_x_1 = const()[name = tensor("op_1830_transpose_x_1"), val = tensor(true)]; + tensor var_1830_transpose_y_1 = const()[name = tensor("op_1830_transpose_y_1"), val = tensor(false)]; + tensor var_1830 = matmul(transpose_x = var_1830_transpose_x_1, transpose_y = var_1830_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1830")]; + tensor new_kv_unnorm = add(x = var_1828, y = var_1830)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1653)[name = tensor("new_scale")]; + tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; + tensor clip_44 = clip(alpha = var_1510, beta = const_33, x = new_scale)[name = tensor("clip_44")]; tensor sqrt_new_scale = sqrt(x = clip_44)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_2699_perm_0 = const()[name = tensor("op_2699_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1839_perm_0 = const()[name = tensor("op_1839_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_2699 = transpose(perm = var_2699_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_2368, x = var_2699)[name = tensor("out_33")]; - tensor var_2703 = const()[name = tensor("op_2703"), val = tensor([9, 5, 256])]; - tensor out = reshape(shape = var_2703, x = out_33)[name = tensor("out")]; - tensor var_2705 = silu(x = input_347)[name = tensor("op_2705")]; - tensor input_349 = mul(x = var_2705, y = out)[name = tensor("input_349")]; - tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_349)[name = tensor("linear_51")]; - tensor input_351 = add(x = x, y = ret_out)[name = tensor("input_351")]; + tensor var_1839 = transpose(perm = var_1839_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1507, x = var_1839)[name = tensor("out_33")]; + tensor var_1843 = const()[name = tensor("op_1843"), val = tensor([9, 5, 256])]; + tensor out = reshape(shape = var_1843, x = out_33)[name = tensor("out")]; + tensor var_1845 = silu(x = input_187)[name = tensor("op_1845")]; + tensor input_189 = mul(x = var_1845, y = out)[name = tensor("input_189")]; + tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; + tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_2366, gamma = decoder_norm11_1_weight, x = input_351)[name = tensor("xt_5")]; - tensor var_2715 = const()[name = tensor("op_2715"), val = tensor([1, 9, 5, 256])]; - tensor var_2716 = reshape(shape = var_2715, x = xt_5)[name = tensor("op_2716")]; - tensor var_2717_perm_0 = const()[name = tensor("op_2717_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2720 = const()[name = tensor("op_2720"), val = tensor([5, 9, 256])]; - tensor var_2717 = transpose(perm = var_2717_perm_0, x = var_2716)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_2720, x = var_2717)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1505, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1855 = const()[name = tensor("op_1855"), val = tensor([1, 9, 5, 256])]; + tensor var_1856 = reshape(shape = var_1855, x = xt_5)[name = tensor("op_1856")]; + tensor var_1857_perm_0 = const()[name = tensor("op_1857_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1860 = const()[name = tensor("op_1860"), val = tensor([5, 9, 256])]; + tensor var_1857 = transpose(perm = var_1857_perm_0, x = var_1856)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1860, x = var_1857)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_2743 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_21 = const()[name = tensor("concat_21"), val = tensor([9, 5, 3, 256])]; - tensor var_2745 = reshape(shape = concat_21, x = var_2743)[name = tensor("op_2745")]; - tensor var_2746_axes_0 = const()[name = tensor("op_2746_axes_0"), val = tensor([0])]; - tensor var_2746 = expand_dims(axes = var_2746_axes_0, x = var_2745)[name = tensor("op_2746")]; - tensor var_2747_perm_0 = const()[name = tensor("op_2747_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2748_axes_0 = const()[name = tensor("op_2748_axes_0"), val = tensor([-2])]; - tensor var_2747 = transpose(perm = var_2747_perm_0, x = var_2746)[name = tensor("transpose_8")]; - tensor var_2748 = squeeze(axes = var_2748_axes_0, x = var_2747)[name = tensor("op_2748")]; + tensor var_1883 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_22 = const()[name = tensor("concat_22"), val = tensor([9, 5, 3, 256])]; + tensor var_1885 = reshape(shape = concat_22, x = var_1883)[name = tensor("op_1885")]; + tensor var_1886_axes_0 = const()[name = tensor("op_1886_axes_0"), val = tensor([0])]; + tensor var_1886 = expand_dims(axes = var_1886_axes_0, x = var_1885)[name = tensor("op_1886")]; + tensor var_1887_perm_0 = const()[name = tensor("op_1887_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1888_axes_0 = const()[name = tensor("op_1888_axes_0"), val = tensor([-2])]; + tensor var_1887 = transpose(perm = var_1887_perm_0, x = var_1886)[name = tensor("transpose_8")]; + tensor var_1888 = squeeze(axes = var_1888_axes_0, x = var_1887)[name = tensor("op_1888")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 9, 5, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_2748)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1888)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 9, 5, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_2748)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1888)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 9, 5, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_2748)[name = tensor("v_19")]; - tensor var_2756 = const()[name = tensor("op_2756"), val = tensor([9, 20, 64])]; - tensor var_2757 = reshape(shape = var_2756, x = q_19)[name = tensor("op_2757")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1888)[name = tensor("v_19")]; + tensor var_1896 = const()[name = tensor("op_1896"), val = tensor([9, 20, 64])]; + tensor var_1897 = reshape(shape = var_1896, x = q_19)[name = tensor("op_1897")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2763 = const()[name = tensor("op_2763"), val = tensor([9, 20, 64])]; - tensor var_2764 = reshape(shape = var_2763, x = k_19)[name = tensor("op_2764")]; + tensor var_1903 = const()[name = tensor("op_1903"), val = tensor([9, 20, 64])]; + tensor var_1904 = reshape(shape = var_1903, x = k_19)[name = tensor("op_1904")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2770 = const()[name = tensor("op_2770"), val = tensor([9, 20, 64])]; - tensor var_2771 = reshape(shape = var_2770, x = v_19)[name = tensor("op_2771")]; + tensor var_1910 = const()[name = tensor("op_1910"), val = tensor([9, 20, 64])]; + tensor var_1911 = reshape(shape = var_1910, x = v_19)[name = tensor("op_1911")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2774 = const()[name = tensor("op_2774"), val = tensor([5, 4, 9, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_2757)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_2774, x = q_21)[name = tensor("q")]; - tensor var_2776 = const()[name = tensor("op_2776"), val = tensor([5, 4, 9, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_2764)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_2776, x = k_21)[name = tensor("k")]; - tensor var_2778 = const()[name = tensor("op_2778"), val = tensor([5, 4, 9, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_2771)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_2778, x = v_21)[name = tensor("v")]; + tensor var_1914 = const()[name = tensor("op_1914"), val = tensor([5, 4, 9, 64])]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1897)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1914, x = q_21)[name = tensor("q")]; + tensor var_1916 = const()[name = tensor("op_1916"), val = tensor([5, 4, 9, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1904)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1916, x = k_21)[name = tensor("k")]; + tensor var_1918 = const()[name = tensor("op_1918"), val = tensor([5, 4, 9, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1911)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1918, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -2439,50 +1824,50 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_2781 = const()[name = tensor("op_2781"), val = tensor([2, 0, 1, 3])]; - tensor var_2786 = const()[name = tensor("op_2786"), val = tensor([45, 256])]; - tensor var_2782 = transpose(perm = var_2781, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_2786, x = var_2782)[name = tensor("attn_output_11")]; + tensor var_1921 = const()[name = tensor("op_1921"), val = tensor([2, 0, 1, 3])]; + tensor var_1926 = const()[name = tensor("op_1926"), val = tensor([45, 256])]; + tensor var_1922 = transpose(perm = var_1921, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1926, x = var_1922)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_2790 = const()[name = tensor("op_2790"), val = tensor([9, 5, 256])]; - tensor attn_output = reshape(shape = var_2790, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1930 = const()[name = tensor("op_1930"), val = tensor([9, 5, 256])]; + tensor attn_output = reshape(shape = var_1930, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; - tensor input_353 = add(x = query_5, y = sa2_out)[name = tensor("input_353")]; - tensor input_355_axes_0 = const()[name = tensor("input_355_axes_0"), val = tensor([-1])]; - tensor input_355 = layer_norm(axes = input_355_axes_0, beta = decoder_norm21_1_bias, epsilon = var_2366, gamma = decoder_norm21_1_weight, x = input_353)[name = tensor("input_355")]; - tensor input_357 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_355)[name = tensor("linear_54")]; - tensor input_359 = relu(x = input_357)[name = tensor("input_359")]; - tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_359)[name = tensor("linear_55")]; - tensor input_361 = add(x = input_355, y = ffn_out)[name = tensor("input_361")]; + tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; + tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1505, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; + tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; + tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; + tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_2366, gamma = decoder_norm22_1_weight, x = input_361)[name = tensor("xt")]; - tensor var_2810 = const()[name = tensor("op_2810"), val = tensor([1, 5, 9, 256])]; - tensor input = reshape(shape = var_2810, x = xt)[name = tensor("input")]; - tensor var_2812 = const()[name = tensor("op_2812"), val = tensor([-1])]; - tensor var_2813 = reduce_l2_norm(axes = var_2812, keep_dims = var_2369, x = input)[name = tensor("op_2813")]; - tensor const_58 = const()[name = tensor("const_58"), val = tensor(0x1.fffffep+127)]; - tensor clip_45 = clip(alpha = var_2360, beta = const_58, x = var_2813)[name = tensor("clip_45")]; - tensor var_2815 = real_div(x = input, y = clip_45)[name = tensor("op_2815")]; - tensor concat_25 = const()[name = tensor("concat_25"), val = tensor([5, 1, 256])]; - tensor reshape_0 = reshape(shape = concat_25, x = emb)[name = tensor("reshape_0")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1505, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1950 = const()[name = tensor("op_1950"), val = tensor([1, 5, 9, 256])]; + tensor input = reshape(shape = var_1950, x = xt)[name = tensor("input")]; + tensor var_1952 = const()[name = tensor("op_1952"), val = tensor([-1])]; + tensor var_1953 = reduce_l2_norm(axes = var_1952, keep_dims = var_1508, x = input)[name = tensor("op_1953")]; + tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; + tensor clip_45 = clip(alpha = var_1500, beta = const_42, x = var_1953)[name = tensor("clip_45")]; + tensor var_1955 = real_div(x = input, y = clip_45)[name = tensor("op_1955")]; + tensor concat_26 = const()[name = tensor("concat_26"), val = tensor([5, 1, 256])]; + tensor reshape_0 = reshape(shape = concat_26, x = emb)[name = tensor("reshape_0")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_26 = const()[name = tensor("concat_26"), val = tensor([5, 256, 9])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_2815)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_26, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_27 = const()[name = tensor("concat_27"), val = tensor([5, 256, 9])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1955)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_27, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = reshape_0, y = reshape_1)[name = tensor("matmul_2")]; - tensor concat_29 = const()[name = tensor("concat_29"), val = tensor([1, 5, 9])]; - tensor reshape_2 = reshape(shape = concat_29, x = matmul_2)[name = tensor("reshape_2")]; + tensor concat_30 = const()[name = tensor("concat_30"), val = tensor([1, 5, 9])]; + tensor reshape_2 = reshape(shape = concat_30, x = matmul_2)[name = tensor("reshape_2")]; tensor output_begin_0 = const()[name = tensor("output_begin_0"), val = tensor([0, 0, 1])]; tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 5, 8])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = reshape_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_2819")]; - tensor var_2821_axis_0 = const()[name = tensor("op_2821_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_2821_axis_0, values = (var_2517, nkv))[name = tensor("op_2821")]; - tensor var_2823_axis_0 = const()[name = tensor("op_2823_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_2823_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_2823")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1959")]; + tensor var_1961_axis_0 = const()[name = tensor("op_1961_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1961_axis_0, values = (var_1657, nkv))[name = tensor("op_1961")]; + tensor var_1963_axis_0 = const()[name = tensor("op_1963_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1963_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1963")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/CALLHOME/ls_eend_ch_500ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/CALLHOME/ls_eend_ch_500ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index 791338ee8b7906d475e86f89ebd7f686aae48d22..94b837f126f7b275b571e4b70ec3ba15241696a4 100644 --- a/optimized/CALLHOME/ls_eend_ch_500ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/CALLHOME/ls_eend_ch_500ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:760f42e2c0837c945eca7fd97195e8902da9c1e6c70c7dacb8dd72dd0c6978e6 -size 352362 +oid sha256:d2370742faddf3b5aed1fed0221c87144bb7e46901fa178dfffec47d80173d5e +size 255672 diff --git a/optimized/CALLHOME/ls_eend_ch_500ms.mlpackage/Manifest.json b/optimized/CALLHOME/ls_eend_ch_500ms.mlpackage/Manifest.json index e2bd5c4d148a1e7e9adf2f8a9cd06c0804b03996..cb936c1e2d3ed54dcbf28b90d4d16ec67441d7dd 100644 --- a/optimized/CALLHOME/ls_eend_ch_500ms.mlpackage/Manifest.json +++ b/optimized/CALLHOME/ls_eend_ch_500ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "BE882AE8-4DFB-4832-B31E-627ED3B0D829": { - "author": "com.apple.CoreML", - "description": "CoreML Model Weights", - "name": "weights", - "path": "com.apple.CoreML/weights" - }, - "D33C57F2-1FF0-4502-8793-91F7365FDCC5": { + "08DB05EC-F148-403B-9053-485F8ADC2A8A": { "author": "com.apple.CoreML", "description": "CoreML Model Specification", "name": "model.mlmodel", "path": "com.apple.CoreML/model.mlmodel" + }, + "2AC932ED-ECDE-4536-B850-B24E81D535F7": { + "author": "com.apple.CoreML", + "description": "CoreML Model Weights", + "name": "weights", + "path": "com.apple.CoreML/weights" } }, - "rootModelIdentifier": "D33C57F2-1FF0-4502-8793-91F7365FDCC5" + "rootModelIdentifier": "08DB05EC-F148-403B-9053-485F8ADC2A8A" } diff --git a/optimized/DIHARD II/.DS_Store b/optimized/DIHARD II/.DS_Store index 4cb8df08847a081542bf3e4c51a1c5ffe61c73bd..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 100644 Binary files a/optimized/DIHARD II/.DS_Store and b/optimized/DIHARD II/.DS_Store differ diff --git a/optimized/DIHARD II/ls_eend_dih2_100ms.mlmodelc/analytics/coremldata.bin b/optimized/DIHARD II/ls_eend_dih2_100ms.mlmodelc/analytics/coremldata.bin index 03a07a98d8dd30c2017ce60b0c3ae9ab8226da7d..052e6ef31fe7d71dbf2275df2ecddef82c17ecda 100644 --- a/optimized/DIHARD II/ls_eend_dih2_100ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/DIHARD II/ls_eend_dih2_100ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ade3526bb524dd257220f9cfb402800c9f1178d780de5e5ae105eb35048f4c80 +oid sha256:6f58876c0a6f0f186408612178ce3af80a3ad107202185cb801ac6d759d17222 size 243 diff --git a/optimized/DIHARD II/ls_eend_dih2_100ms.mlmodelc/coremldata.bin b/optimized/DIHARD II/ls_eend_dih2_100ms.mlmodelc/coremldata.bin index 2898c9d90ab0480bace8ed6c4f68303df57adadd..2a7d96e4b78c57bfb0d73d9fd71cec30b254f52b 100644 --- a/optimized/DIHARD II/ls_eend_dih2_100ms.mlmodelc/coremldata.bin +++ b/optimized/DIHARD II/ls_eend_dih2_100ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f09632ad9eb8a41b2746e69404a2ecf88a9b5007d76281ffa854deba8ec37a67 -size 1266 +oid sha256:15fbad4bd08b4a58160983abab0889fe5ca399347f80c5aa5c7234d4bac9c1fb +size 1357 diff --git a/optimized/DIHARD II/ls_eend_dih2_100ms.mlmodelc/metadata.json b/optimized/DIHARD II/ls_eend_dih2_100ms.mlmodelc/metadata.json index 80856805940bcde2803d5168893cd9b25b313a58..8f707d581e42c3a2970d1ee7b8cb0818c2e09b40 100644 --- a/optimized/DIHARD II/ls_eend_dih2_100ms.mlmodelc/metadata.json +++ b/optimized/DIHARD II/ls_eend_dih2_100ms.mlmodelc/metadata.json @@ -96,7 +96,7 @@ "Ios17.reshape" : 74, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 53, + "Ios17.transpose" : 57, "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 58, @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"dih2\", \"model_label\": \"DIHARD II\", \"variant\": \"pipeline\", \"T\": 1, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"dih2\", \"model_label\": \"DIHARD II\", \"variant\": \"pipeline\", \"chunk_size\": 1, \"step_duration_ms\": 100, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/DIHARD II/ls_eend_dih2_100ms.mlmodelc/model.mil b/optimized/DIHARD II/ls_eend_dih2_100ms.mlmodelc/model.mil index 742a1c536563c04e1f971f8b28aa14c6f74c6dfa..a20414bfdbdabb07fc1d4dae565285b934f5f3a5 100644 --- a/optimized/DIHARD II/ls_eend_dih2_100ms.mlmodelc/model.mil +++ b/optimized/DIHARD II/ls_eend_dih2_100ms.mlmodelc/model.mil @@ -216,6 +216,7 @@ program(1.0) tensor var_21 = const()[name = tensor("op_21"), val = tensor(0x0p+0)]; tensor var_24 = const()[name = tensor("op_24"), val = tensor(0x1.0c6f7ap-20)]; tensor var_27 = const()[name = tensor("op_27"), val = tensor(2)]; + tensor var_30 = const()[name = tensor("op_30"), val = tensor(0)]; tensor var_33 = const()[name = tensor("op_33"), val = tensor(1)]; tensor var_35 = const()[name = tensor("op_35"), val = tensor(0x1.4f8b58p-17)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(true)]; @@ -262,8 +263,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_53")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_54")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_196 = const()[name = tensor("op_196"), val = tensor([1, 1])]; tensor var_197 = reshape(shape = var_196, x = sqrt_s_t_1)[name = tensor("op_197")]; @@ -271,7 +272,7 @@ program(1.0) tensor var_199 = mul(x = qk_1, y = M_1)[name = tensor("op_199")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_52")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_199, y = v_1)[name = tensor("inner_1")]; tensor var_201_transpose_x_0 = const()[name = tensor("op_201_transpose_x_0"), val = tensor(false)]; tensor var_201_transpose_y_0 = const()[name = tensor("op_201_transpose_y_0"), val = tensor(false)]; @@ -292,7 +293,7 @@ program(1.0) tensor var_214 = real_div(x = new_kv_unnorm_1, y = var_213)[name = tensor("op_214")]; tensor var_215_perm_0 = const()[name = tensor("op_215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_51")]; + tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_24, x = var_215)[name = tensor("out_3")]; tensor var_219 = const()[name = tensor("op_219"), val = tensor([1, 1, 256])]; tensor out_5 = reshape(shape = var_219, x = out_3)[name = tensor("out_5")]; @@ -336,10 +337,12 @@ program(1.0) tensor var_252 = sub(x = var_17, y = use_roll_1)[name = tensor("op_252")]; tensor var_253 = mul(x = filled_window_1, y = var_252)[name = tensor("op_253")]; tensor var_254 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_254")]; - tensor input_21 = add(x = var_253, y = var_254)[name = tensor("input_21")]; + tensor window_3 = add(x = var_253, y = var_254)[name = tensor("window_3")]; tensor var_256 = const()[name = tensor("op_256"), val = tensor(0x1p+0)]; tensor var_257 = add(x = filled_1, y = var_256)[name = tensor("op_257")]; tensor clip_1 = clip(alpha = var_21, beta = var_16, x = var_257)[name = tensor("clip_1")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_30, interleave = input_21_interleave_0, values = window_3)[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; @@ -348,13 +351,13 @@ program(1.0) tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_50")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_278_split_sizes_0 = const()[name = tensor("op_278_split_sizes_0"), val = tensor([256, 256])]; - tensor var_278_axis_0 = const()[name = tensor("op_278_axis_0"), val = tensor(1)]; - tensor var_278_0, tensor var_278_1 = split(axis = var_278_axis_0, split_sizes = var_278_split_sizes_0, x = inputs_3)[name = tensor("op_278")]; - tensor var_280 = sigmoid(x = var_278_1)[name = tensor("op_280")]; - tensor inputs_5 = mul(x = var_278_0, y = var_280)[name = tensor("inputs_5")]; + tensor var_280_split_sizes_0 = const()[name = tensor("op_280_split_sizes_0"), val = tensor([256, 256])]; + tensor var_280_axis_0 = const()[name = tensor("op_280_axis_0"), val = tensor(1)]; + tensor var_280_0, tensor var_280_1 = split(axis = var_280_axis_0, split_sizes = var_280_split_sizes_0, x = inputs_3)[name = tensor("op_280")]; + tensor var_282 = sigmoid(x = var_280_1)[name = tensor("op_282")]; + tensor inputs_5 = mul(x = var_280_0, y = var_282)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; @@ -374,22 +377,22 @@ program(1.0) tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_311_begin_0 = const()[name = tensor("op_311_begin_0"), val = tensor([0, -1, 0])]; - tensor var_311_end_0 = const()[name = tensor("op_311_end_0"), val = tensor([1, 16, 256])]; - tensor var_311_end_mask_0 = const()[name = tensor("op_311_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_49")]; - tensor var_311 = slice_by_index(begin = var_311_begin_0, end = var_311_end_0, end_mask = var_311_end_mask_0, x = conv_out_1)[name = tensor("op_311")]; - tensor var_314_interleave_0 = const()[name = tensor("op_314_interleave_0"), val = tensor(false)]; - tensor var_314 = concat(axis = var_33, interleave = var_314_interleave_0, values = var_311)[name = tensor("op_314")]; - tensor input_31 = add(x = x_3, y = var_314)[name = tensor("input_31")]; + tensor var_313_begin_0 = const()[name = tensor("op_313_begin_0"), val = tensor([0, -1, 0])]; + tensor var_313_end_0 = const()[name = tensor("op_313_end_0"), val = tensor([1, 16, 256])]; + tensor var_313_end_mask_0 = const()[name = tensor("op_313_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_313 = slice_by_index(begin = var_313_begin_0, end = var_313_end_0, end_mask = var_313_end_mask_0, x = conv_out_1)[name = tensor("op_313")]; + tensor var_315_perm_0 = const()[name = tensor("op_315_perm_0"), val = tensor([1, 0, 2])]; + tensor var_315 = transpose(perm = var_315_perm_0, x = var_313)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_315)[name = tensor("input_31")]; tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; - tensor var_337 = const()[name = tensor("op_337"), val = tensor(0x1p-1)]; - tensor var_338 = mul(x = input_39, y = var_337)[name = tensor("op_338")]; - tensor input_41 = add(x = var_338, y = input_31)[name = tensor("input_41")]; + tensor var_338 = const()[name = tensor("op_338"), val = tensor(0x1p-1)]; + tensor var_339 = mul(x = input_39, y = var_338)[name = tensor("op_339")]; + tensor input_41 = add(x = var_339, y = input_31)[name = tensor("input_41")]; tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; @@ -397,9 +400,9 @@ program(1.0) tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; - tensor var_367 = const()[name = tensor("op_367"), val = tensor(0x1p-1)]; - tensor var_368 = mul(x = input_51, y = var_367)[name = tensor("op_368")]; - tensor input_53 = add(x = var_368, y = input_43)[name = tensor("input_53")]; + tensor var_368 = const()[name = tensor("op_368"), val = tensor(0x1p-1)]; + tensor var_369 = mul(x = input_51, y = var_368)[name = tensor("op_369")]; + tensor input_53 = add(x = var_369, y = input_43)[name = tensor("input_53")]; tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; @@ -412,19 +415,19 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_382 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; - tensor var_383 = const()[name = tensor("op_383"), val = tensor([1, 1, 4, 64])]; - tensor var_384 = reshape(shape = var_383, x = var_382)[name = tensor("op_384")]; + tensor var_383 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_384 = const()[name = tensor("op_384"), val = tensor([1, 1, 4, 64])]; + tensor var_385 = reshape(shape = var_384, x = var_383)[name = tensor("op_385")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_388 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; - tensor var_389 = const()[name = tensor("op_389"), val = tensor(0x1p-3)]; - tensor var_390 = mul(x = var_388, y = var_389)[name = tensor("op_390")]; - tensor var_391 = const()[name = tensor("op_391"), val = tensor([1, 1, 4, 64])]; - tensor var_392 = reshape(shape = var_391, x = var_390)[name = tensor("op_392")]; + tensor var_389 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_390 = const()[name = tensor("op_390"), val = tensor(0x1p-3)]; + tensor var_391 = mul(x = var_389, y = var_390)[name = tensor("op_391")]; + tensor var_392 = const()[name = tensor("op_392"), val = tensor([1, 1, 4, 64])]; + tensor var_393 = reshape(shape = var_392, x = var_391)[name = tensor("op_393")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_396 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; - tensor var_397 = const()[name = tensor("op_397"), val = tensor([1, 1, 4, 64])]; - tensor var_398 = reshape(shape = var_397, x = var_396)[name = tensor("op_398")]; + tensor var_397 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_398 = const()[name = tensor("op_398"), val = tensor([1, 1, 4, 64])]; + tensor var_399 = reshape(shape = var_398, x = var_397)[name = tensor("op_399")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; @@ -432,84 +435,86 @@ program(1.0) tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_392)[name = tensor("transpose_47")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_384)[name = tensor("transpose_48")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_393)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_385)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_408 = const()[name = tensor("op_408"), val = tensor([1, 1])]; - tensor var_409 = reshape(shape = var_408, x = sqrt_s_t_3)[name = tensor("op_409")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_409)[name = tensor("M_3")]; - tensor var_411 = mul(x = qk_3, y = M_3)[name = tensor("op_411")]; + tensor var_409 = const()[name = tensor("op_409"), val = tensor([1, 1])]; + tensor var_410 = reshape(shape = var_409, x = sqrt_s_t_3)[name = tensor("op_410")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_410)[name = tensor("M_3")]; + tensor var_412 = mul(x = qk_3, y = M_3)[name = tensor("op_412")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_398)[name = tensor("transpose_46")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_411, y = v_3)[name = tensor("inner_3")]; - tensor var_413_transpose_x_0 = const()[name = tensor("op_413_transpose_x_0"), val = tensor(false)]; - tensor var_413_transpose_y_0 = const()[name = tensor("op_413_transpose_y_0"), val = tensor(false)]; - tensor var_413 = matmul(transpose_x = var_413_transpose_x_0, transpose_y = var_413_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_413")]; - tensor var_414 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_414")]; - tensor var_415 = const()[name = tensor("op_415"), val = tensor([1, 1, 1, 1])]; - tensor var_416 = reshape(shape = var_415, x = var_414)[name = tensor("op_416")]; - tensor cross_3 = mul(x = var_413, y = var_416)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_399)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_412, y = v_3)[name = tensor("inner_3")]; + tensor var_414_transpose_x_0 = const()[name = tensor("op_414_transpose_x_0"), val = tensor(false)]; + tensor var_414_transpose_y_0 = const()[name = tensor("op_414_transpose_y_0"), val = tensor(false)]; + tensor var_414 = matmul(transpose_x = var_414_transpose_x_0, transpose_y = var_414_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_414")]; + tensor var_415 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_415")]; + tensor var_416 = const()[name = tensor("op_416"), val = tensor([1, 1, 1, 1])]; + tensor var_417 = reshape(shape = var_416, x = var_415)[name = tensor("op_417")]; + tensor cross_3 = mul(x = var_414, y = var_417)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_419 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_419")]; - tensor var_421_transpose_x_1 = const()[name = tensor("op_421_transpose_x_1"), val = tensor(true)]; - tensor var_421_transpose_y_1 = const()[name = tensor("op_421_transpose_y_1"), val = tensor(false)]; - tensor var_421 = matmul(transpose_x = var_421_transpose_x_1, transpose_y = var_421_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_421")]; - tensor new_kv_unnorm_3 = add(x = var_419, y = var_421)[name = tensor("new_kv_unnorm_3")]; - tensor var_423 = const()[name = tensor("op_423"), val = tensor(0x1p+0)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_423)[name = tensor("new_scale_3")]; - tensor var_425 = sqrt(x = new_scale_3)[name = tensor("op_425")]; - tensor var_426 = real_div(x = new_kv_unnorm_3, y = var_425)[name = tensor("op_426")]; - tensor var_427_perm_0 = const()[name = tensor("op_427_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_420 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_420")]; + tensor var_422_transpose_x_1 = const()[name = tensor("op_422_transpose_x_1"), val = tensor(true)]; + tensor var_422_transpose_y_1 = const()[name = tensor("op_422_transpose_y_1"), val = tensor(false)]; + tensor var_422 = matmul(transpose_x = var_422_transpose_x_1, transpose_y = var_422_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_422")]; + tensor new_kv_unnorm_3 = add(x = var_420, y = var_422)[name = tensor("new_kv_unnorm_3")]; + tensor var_424 = const()[name = tensor("op_424"), val = tensor(0x1p+0)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_424)[name = tensor("new_scale_3")]; + tensor var_426 = sqrt(x = new_scale_3)[name = tensor("op_426")]; + tensor var_427 = real_div(x = new_kv_unnorm_3, y = var_426)[name = tensor("op_427")]; + tensor var_428_perm_0 = const()[name = tensor("op_428_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_427 = transpose(perm = var_427_perm_0, x = out_7)[name = tensor("transpose_45")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_427)[name = tensor("out_9")]; - tensor var_431 = const()[name = tensor("op_431"), val = tensor([1, 1, 256])]; - tensor out_11 = reshape(shape = var_431, x = out_9)[name = tensor("out_11")]; - tensor var_433 = silu(x = input_57)[name = tensor("op_433")]; - tensor input_59 = mul(x = var_433, y = out_11)[name = tensor("input_59")]; + tensor var_428 = transpose(perm = var_428_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_428)[name = tensor("out_9")]; + tensor var_432 = const()[name = tensor("op_432"), val = tensor([1, 1, 256])]; + tensor out_11 = reshape(shape = var_432, x = out_9)[name = tensor("out_11")]; + tensor var_434 = silu(x = input_57)[name = tensor("op_434")]; + tensor input_59 = mul(x = var_434, y = out_11)[name = tensor("input_59")]; tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; tensor filled_3_begin_0 = const()[name = tensor("filled_3_begin_0"), val = tensor([1, 0])]; tensor filled_3_end_0 = const()[name = tensor("filled_3_end_0"), val = tensor([2, 1])]; tensor filled_3_end_mask_0 = const()[name = tensor("filled_3_end_mask_0"), val = tensor([false, true])]; tensor filled_3_squeeze_mask_0 = const()[name = tensor("filled_3_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_3 = slice_by_index(begin = filled_3_begin_0, end = filled_3_end_0, end_mask = filled_3_end_mask_0, squeeze_mask = filled_3_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_3")]; - tensor var_445_begin_0 = const()[name = tensor("op_445_begin_0"), val = tensor([0, 1, 0])]; - tensor var_445_end_0 = const()[name = tensor("op_445_end_0"), val = tensor([1, 16, 256])]; - tensor var_445_end_mask_0 = const()[name = tensor("op_445_end_mask_0"), val = tensor([true, true, true])]; - tensor var_445 = slice_by_index(begin = var_445_begin_0, end = var_445_end_0, end_mask = var_445_end_mask_0, x = window_3)[name = tensor("op_445")]; + tensor var_446_begin_0 = const()[name = tensor("op_446_begin_0"), val = tensor([0, 1, 0])]; + tensor var_446_end_0 = const()[name = tensor("op_446_end_0"), val = tensor([1, 16, 256])]; + tensor var_446_end_mask_0 = const()[name = tensor("op_446_end_mask_0"), val = tensor([true, true, true])]; + tensor var_446 = slice_by_index(begin = var_446_begin_0, end = var_446_end_0, end_mask = var_446_end_mask_0, x = window_5)[name = tensor("op_446")]; tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_445, x_9))[name = tensor("rolled_3")]; + tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_446, x_9))[name = tensor("rolled_3")]; tensor clip_2 = clip(alpha = var_21, beta = var_20, x = filled_3)[name = tensor("clip_2")]; - tensor var_450 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_450")]; - tensor var_451_dtype_0 = const()[name = tensor("op_451_dtype_0"), val = tensor("fp32")]; - tensor var_452 = const()[name = tensor("op_452"), val = tensor([1, 16, 1])]; - tensor var_451 = cast(dtype = var_451_dtype_0, x = var_450)[name = tensor("cast_125")]; - tensor fill_mask_3 = reshape(shape = var_452, x = var_451)[name = tensor("fill_mask_3")]; + tensor var_451 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_451")]; + tensor var_452_dtype_0 = const()[name = tensor("op_452_dtype_0"), val = tensor("fp32")]; + tensor var_453 = const()[name = tensor("op_453"), val = tensor([1, 16, 1])]; + tensor var_452 = cast(dtype = var_452_dtype_0, x = var_451)[name = tensor("cast_125")]; + tensor fill_mask_3 = reshape(shape = var_453, x = var_452)[name = tensor("fill_mask_3")]; tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = x_9)[name = tensor("x_expanded_3")]; - tensor var_456 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_456")]; - tensor var_457 = mul(x = window_3, y = var_456)[name = tensor("op_457")]; - tensor var_458 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_458")]; - tensor filled_window_3 = add(x = var_457, y = var_458)[name = tensor("filled_window_3")]; - tensor var_460 = greater_equal(x = filled_3, y = var_16)[name = tensor("op_460")]; - tensor var_461_dtype_0 = const()[name = tensor("op_461_dtype_0"), val = tensor("fp32")]; - tensor var_462 = const()[name = tensor("op_462"), val = tensor([1, 1, 1])]; - tensor var_461 = cast(dtype = var_461_dtype_0, x = var_460)[name = tensor("cast_124")]; - tensor use_roll_3 = reshape(shape = var_462, x = var_461)[name = tensor("use_roll_3")]; - tensor var_464 = sub(x = var_17, y = use_roll_3)[name = tensor("op_464")]; - tensor var_465 = mul(x = filled_window_3, y = var_464)[name = tensor("op_465")]; - tensor var_466 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_466")]; - tensor input_61 = add(x = var_465, y = var_466)[name = tensor("input_61")]; - tensor var_468 = const()[name = tensor("op_468"), val = tensor(0x1p+0)]; - tensor var_469 = add(x = filled_3, y = var_468)[name = tensor("op_469")]; - tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_469)[name = tensor("clip_3")]; + tensor var_457 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_457")]; + tensor var_458 = mul(x = window_5, y = var_457)[name = tensor("op_458")]; + tensor var_459 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_459")]; + tensor filled_window_3 = add(x = var_458, y = var_459)[name = tensor("filled_window_3")]; + tensor var_461 = greater_equal(x = filled_3, y = var_16)[name = tensor("op_461")]; + tensor var_462_dtype_0 = const()[name = tensor("op_462_dtype_0"), val = tensor("fp32")]; + tensor var_463 = const()[name = tensor("op_463"), val = tensor([1, 1, 1])]; + tensor var_462 = cast(dtype = var_462_dtype_0, x = var_461)[name = tensor("cast_124")]; + tensor use_roll_3 = reshape(shape = var_463, x = var_462)[name = tensor("use_roll_3")]; + tensor var_465 = sub(x = var_17, y = use_roll_3)[name = tensor("op_465")]; + tensor var_466 = mul(x = filled_window_3, y = var_465)[name = tensor("op_466")]; + tensor var_467 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_467")]; + tensor window_7 = add(x = var_466, y = var_467)[name = tensor("window_7")]; + tensor var_469 = const()[name = tensor("op_469"), val = tensor(0x1p+0)]; + tensor var_470 = add(x = filled_3, y = var_469)[name = tensor("op_470")]; + tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_470)[name = tensor("clip_3")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_30, interleave = input_61_interleave_0, values = window_7)[name = tensor("input_61")]; tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; @@ -518,13 +523,13 @@ program(1.0) tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; - tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_44")]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; - tensor var_490_split_sizes_0 = const()[name = tensor("op_490_split_sizes_0"), val = tensor([256, 256])]; - tensor var_490_axis_0 = const()[name = tensor("op_490_axis_0"), val = tensor(1)]; - tensor var_490_0, tensor var_490_1 = split(axis = var_490_axis_0, split_sizes = var_490_split_sizes_0, x = inputs_13)[name = tensor("op_490")]; - tensor var_492 = sigmoid(x = var_490_1)[name = tensor("op_492")]; - tensor inputs_15 = mul(x = var_490_0, y = var_492)[name = tensor("inputs_15")]; + tensor var_493_split_sizes_0 = const()[name = tensor("op_493_split_sizes_0"), val = tensor([256, 256])]; + tensor var_493_axis_0 = const()[name = tensor("op_493_axis_0"), val = tensor(1)]; + tensor var_493_0, tensor var_493_1 = split(axis = var_493_axis_0, split_sizes = var_493_split_sizes_0, x = inputs_13)[name = tensor("op_493")]; + tensor var_495 = sigmoid(x = var_493_1)[name = tensor("op_495")]; + tensor inputs_15 = mul(x = var_493_0, y = var_495)[name = tensor("inputs_15")]; tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; @@ -544,22 +549,22 @@ program(1.0) tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_523_begin_0 = const()[name = tensor("op_523_begin_0"), val = tensor([0, -1, 0])]; - tensor var_523_end_0 = const()[name = tensor("op_523_end_0"), val = tensor([1, 16, 256])]; - tensor var_523_end_mask_0 = const()[name = tensor("op_523_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_43")]; - tensor var_523 = slice_by_index(begin = var_523_begin_0, end = var_523_end_0, end_mask = var_523_end_mask_0, x = conv_out_3)[name = tensor("op_523")]; - tensor var_526_interleave_0 = const()[name = tensor("op_526_interleave_0"), val = tensor(false)]; - tensor var_526 = concat(axis = var_33, interleave = var_526_interleave_0, values = var_523)[name = tensor("op_526")]; - tensor input_71 = add(x = x_9, y = var_526)[name = tensor("input_71")]; + tensor var_526_begin_0 = const()[name = tensor("op_526_begin_0"), val = tensor([0, -1, 0])]; + tensor var_526_end_0 = const()[name = tensor("op_526_end_0"), val = tensor([1, 16, 256])]; + tensor var_526_end_mask_0 = const()[name = tensor("op_526_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_526 = slice_by_index(begin = var_526_begin_0, end = var_526_end_0, end_mask = var_526_end_mask_0, x = conv_out_3)[name = tensor("op_526")]; + tensor var_528_perm_0 = const()[name = tensor("op_528_perm_0"), val = tensor([1, 0, 2])]; + tensor var_528 = transpose(perm = var_528_perm_0, x = var_526)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_528)[name = tensor("input_71")]; tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; - tensor var_549 = const()[name = tensor("op_549"), val = tensor(0x1p-1)]; - tensor var_550 = mul(x = input_79, y = var_549)[name = tensor("op_550")]; - tensor input_81 = add(x = var_550, y = input_71)[name = tensor("input_81")]; + tensor var_551 = const()[name = tensor("op_551"), val = tensor(0x1p-1)]; + tensor var_552 = mul(x = input_79, y = var_551)[name = tensor("op_552")]; + tensor input_81 = add(x = var_552, y = input_71)[name = tensor("input_81")]; tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; @@ -567,9 +572,9 @@ program(1.0) tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; - tensor var_579 = const()[name = tensor("op_579"), val = tensor(0x1p-1)]; - tensor var_580 = mul(x = input_91, y = var_579)[name = tensor("op_580")]; - tensor input_93 = add(x = var_580, y = input_83)[name = tensor("input_93")]; + tensor var_581 = const()[name = tensor("op_581"), val = tensor(0x1p-1)]; + tensor var_582 = mul(x = input_91, y = var_581)[name = tensor("op_582")]; + tensor input_93 = add(x = var_582, y = input_83)[name = tensor("input_93")]; tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; @@ -582,19 +587,19 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_594 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; - tensor var_595 = const()[name = tensor("op_595"), val = tensor([1, 1, 4, 64])]; - tensor var_596 = reshape(shape = var_595, x = var_594)[name = tensor("op_596")]; + tensor var_596 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_597 = const()[name = tensor("op_597"), val = tensor([1, 1, 4, 64])]; + tensor var_598 = reshape(shape = var_597, x = var_596)[name = tensor("op_598")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_600 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; - tensor var_601 = const()[name = tensor("op_601"), val = tensor(0x1p-3)]; - tensor var_602 = mul(x = var_600, y = var_601)[name = tensor("op_602")]; - tensor var_603 = const()[name = tensor("op_603"), val = tensor([1, 1, 4, 64])]; - tensor var_604 = reshape(shape = var_603, x = var_602)[name = tensor("op_604")]; + tensor var_602 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_603 = const()[name = tensor("op_603"), val = tensor(0x1p-3)]; + tensor var_604 = mul(x = var_602, y = var_603)[name = tensor("op_604")]; + tensor var_605 = const()[name = tensor("op_605"), val = tensor([1, 1, 4, 64])]; + tensor var_606 = reshape(shape = var_605, x = var_604)[name = tensor("op_606")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_608 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; - tensor var_609 = const()[name = tensor("op_609"), val = tensor([1, 1, 4, 64])]; - tensor var_610 = reshape(shape = var_609, x = var_608)[name = tensor("op_610")]; + tensor var_610 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_611 = const()[name = tensor("op_611"), val = tensor([1, 1, 4, 64])]; + tensor var_612 = reshape(shape = var_611, x = var_610)[name = tensor("op_612")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; @@ -602,84 +607,86 @@ program(1.0) tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_604)[name = tensor("transpose_41")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_596)[name = tensor("transpose_42")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_606)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_598)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_620 = const()[name = tensor("op_620"), val = tensor([1, 1])]; - tensor var_621 = reshape(shape = var_620, x = sqrt_s_t_5)[name = tensor("op_621")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_621)[name = tensor("M_5")]; - tensor var_623 = mul(x = qk_5, y = M_5)[name = tensor("op_623")]; + tensor var_622 = const()[name = tensor("op_622"), val = tensor([1, 1])]; + tensor var_623 = reshape(shape = var_622, x = sqrt_s_t_5)[name = tensor("op_623")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_623)[name = tensor("M_5")]; + tensor var_625 = mul(x = qk_5, y = M_5)[name = tensor("op_625")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_610)[name = tensor("transpose_40")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_623, y = v_5)[name = tensor("inner_5")]; - tensor var_625_transpose_x_0 = const()[name = tensor("op_625_transpose_x_0"), val = tensor(false)]; - tensor var_625_transpose_y_0 = const()[name = tensor("op_625_transpose_y_0"), val = tensor(false)]; - tensor var_625 = matmul(transpose_x = var_625_transpose_x_0, transpose_y = var_625_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_625")]; - tensor var_626 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_626")]; - tensor var_627 = const()[name = tensor("op_627"), val = tensor([1, 1, 1, 1])]; - tensor var_628 = reshape(shape = var_627, x = var_626)[name = tensor("op_628")]; - tensor cross_5 = mul(x = var_625, y = var_628)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_612)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_625, y = v_5)[name = tensor("inner_5")]; + tensor var_627_transpose_x_0 = const()[name = tensor("op_627_transpose_x_0"), val = tensor(false)]; + tensor var_627_transpose_y_0 = const()[name = tensor("op_627_transpose_y_0"), val = tensor(false)]; + tensor var_627 = matmul(transpose_x = var_627_transpose_x_0, transpose_y = var_627_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_627")]; + tensor var_628 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_628")]; + tensor var_629 = const()[name = tensor("op_629"), val = tensor([1, 1, 1, 1])]; + tensor var_630 = reshape(shape = var_629, x = var_628)[name = tensor("op_630")]; + tensor cross_5 = mul(x = var_627, y = var_630)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_631 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_631")]; - tensor var_633_transpose_x_1 = const()[name = tensor("op_633_transpose_x_1"), val = tensor(true)]; - tensor var_633_transpose_y_1 = const()[name = tensor("op_633_transpose_y_1"), val = tensor(false)]; - tensor var_633 = matmul(transpose_x = var_633_transpose_x_1, transpose_y = var_633_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_633")]; - tensor new_kv_unnorm_5 = add(x = var_631, y = var_633)[name = tensor("new_kv_unnorm_5")]; - tensor var_635 = const()[name = tensor("op_635"), val = tensor(0x1p+0)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_635)[name = tensor("new_scale_5")]; - tensor var_637 = sqrt(x = new_scale_5)[name = tensor("op_637")]; - tensor var_638 = real_div(x = new_kv_unnorm_5, y = var_637)[name = tensor("op_638")]; - tensor var_639_perm_0 = const()[name = tensor("op_639_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_633 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_633")]; + tensor var_635_transpose_x_1 = const()[name = tensor("op_635_transpose_x_1"), val = tensor(true)]; + tensor var_635_transpose_y_1 = const()[name = tensor("op_635_transpose_y_1"), val = tensor(false)]; + tensor var_635 = matmul(transpose_x = var_635_transpose_x_1, transpose_y = var_635_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_635")]; + tensor new_kv_unnorm_5 = add(x = var_633, y = var_635)[name = tensor("new_kv_unnorm_5")]; + tensor var_637 = const()[name = tensor("op_637"), val = tensor(0x1p+0)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_637)[name = tensor("new_scale_5")]; + tensor var_639 = sqrt(x = new_scale_5)[name = tensor("op_639")]; + tensor var_640 = real_div(x = new_kv_unnorm_5, y = var_639)[name = tensor("op_640")]; + tensor var_641_perm_0 = const()[name = tensor("op_641_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_639 = transpose(perm = var_639_perm_0, x = out_13)[name = tensor("transpose_39")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_639)[name = tensor("out_15")]; - tensor var_643 = const()[name = tensor("op_643"), val = tensor([1, 1, 256])]; - tensor out_17 = reshape(shape = var_643, x = out_15)[name = tensor("out_17")]; - tensor var_645 = silu(x = input_97)[name = tensor("op_645")]; - tensor input_99 = mul(x = var_645, y = out_17)[name = tensor("input_99")]; + tensor var_641 = transpose(perm = var_641_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_641)[name = tensor("out_15")]; + tensor var_645 = const()[name = tensor("op_645"), val = tensor([1, 1, 256])]; + tensor out_17 = reshape(shape = var_645, x = out_15)[name = tensor("out_17")]; + tensor var_647 = silu(x = input_97)[name = tensor("op_647")]; + tensor input_99 = mul(x = var_647, y = out_17)[name = tensor("input_99")]; tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor window_9_begin_0 = const()[name = tensor("window_9_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_9_end_0 = const()[name = tensor("window_9_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_9_end_mask_0 = const()[name = tensor("window_9_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_9_squeeze_mask_0 = const()[name = tensor("window_9_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_9 = slice_by_index(begin = window_9_begin_0, end = window_9_end_0, end_mask = window_9_end_mask_0, squeeze_mask = window_9_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_9")]; tensor filled_5_begin_0 = const()[name = tensor("filled_5_begin_0"), val = tensor([2, 0])]; tensor filled_5_end_0 = const()[name = tensor("filled_5_end_0"), val = tensor([3, 1])]; tensor filled_5_end_mask_0 = const()[name = tensor("filled_5_end_mask_0"), val = tensor([false, true])]; tensor filled_5_squeeze_mask_0 = const()[name = tensor("filled_5_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_5 = slice_by_index(begin = filled_5_begin_0, end = filled_5_end_0, end_mask = filled_5_end_mask_0, squeeze_mask = filled_5_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_5")]; - tensor var_657_begin_0 = const()[name = tensor("op_657_begin_0"), val = tensor([0, 1, 0])]; - tensor var_657_end_0 = const()[name = tensor("op_657_end_0"), val = tensor([1, 16, 256])]; - tensor var_657_end_mask_0 = const()[name = tensor("op_657_end_mask_0"), val = tensor([true, true, true])]; - tensor var_657 = slice_by_index(begin = var_657_begin_0, end = var_657_end_0, end_mask = var_657_end_mask_0, x = window_5)[name = tensor("op_657")]; + tensor var_659_begin_0 = const()[name = tensor("op_659_begin_0"), val = tensor([0, 1, 0])]; + tensor var_659_end_0 = const()[name = tensor("op_659_end_0"), val = tensor([1, 16, 256])]; + tensor var_659_end_mask_0 = const()[name = tensor("op_659_end_mask_0"), val = tensor([true, true, true])]; + tensor var_659 = slice_by_index(begin = var_659_begin_0, end = var_659_end_0, end_mask = var_659_end_mask_0, x = window_9)[name = tensor("op_659")]; tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_657, x_15))[name = tensor("rolled_5")]; + tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_659, x_15))[name = tensor("rolled_5")]; tensor clip_4 = clip(alpha = var_21, beta = var_20, x = filled_5)[name = tensor("clip_4")]; - tensor var_662 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_662")]; - tensor var_663_dtype_0 = const()[name = tensor("op_663_dtype_0"), val = tensor("fp32")]; - tensor var_664 = const()[name = tensor("op_664"), val = tensor([1, 16, 1])]; - tensor var_663 = cast(dtype = var_663_dtype_0, x = var_662)[name = tensor("cast_123")]; - tensor fill_mask_5 = reshape(shape = var_664, x = var_663)[name = tensor("fill_mask_5")]; + tensor var_664 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_664")]; + tensor var_665_dtype_0 = const()[name = tensor("op_665_dtype_0"), val = tensor("fp32")]; + tensor var_666 = const()[name = tensor("op_666"), val = tensor([1, 16, 1])]; + tensor var_665 = cast(dtype = var_665_dtype_0, x = var_664)[name = tensor("cast_123")]; + tensor fill_mask_5 = reshape(shape = var_666, x = var_665)[name = tensor("fill_mask_5")]; tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = x_15)[name = tensor("x_expanded_5")]; - tensor var_668 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_668")]; - tensor var_669 = mul(x = window_5, y = var_668)[name = tensor("op_669")]; - tensor var_670 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_670")]; - tensor filled_window_5 = add(x = var_669, y = var_670)[name = tensor("filled_window_5")]; - tensor var_672 = greater_equal(x = filled_5, y = var_16)[name = tensor("op_672")]; - tensor var_673_dtype_0 = const()[name = tensor("op_673_dtype_0"), val = tensor("fp32")]; - tensor var_674 = const()[name = tensor("op_674"), val = tensor([1, 1, 1])]; - tensor var_673 = cast(dtype = var_673_dtype_0, x = var_672)[name = tensor("cast_122")]; - tensor use_roll_5 = reshape(shape = var_674, x = var_673)[name = tensor("use_roll_5")]; - tensor var_676 = sub(x = var_17, y = use_roll_5)[name = tensor("op_676")]; - tensor var_677 = mul(x = filled_window_5, y = var_676)[name = tensor("op_677")]; - tensor var_678 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_678")]; - tensor input_101 = add(x = var_677, y = var_678)[name = tensor("input_101")]; - tensor var_680 = const()[name = tensor("op_680"), val = tensor(0x1p+0)]; - tensor var_681 = add(x = filled_5, y = var_680)[name = tensor("op_681")]; - tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_681)[name = tensor("clip_5")]; + tensor var_670 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_670")]; + tensor var_671 = mul(x = window_9, y = var_670)[name = tensor("op_671")]; + tensor var_672 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_672")]; + tensor filled_window_5 = add(x = var_671, y = var_672)[name = tensor("filled_window_5")]; + tensor var_674 = greater_equal(x = filled_5, y = var_16)[name = tensor("op_674")]; + tensor var_675_dtype_0 = const()[name = tensor("op_675_dtype_0"), val = tensor("fp32")]; + tensor var_676 = const()[name = tensor("op_676"), val = tensor([1, 1, 1])]; + tensor var_675 = cast(dtype = var_675_dtype_0, x = var_674)[name = tensor("cast_122")]; + tensor use_roll_5 = reshape(shape = var_676, x = var_675)[name = tensor("use_roll_5")]; + tensor var_678 = sub(x = var_17, y = use_roll_5)[name = tensor("op_678")]; + tensor var_679 = mul(x = filled_window_5, y = var_678)[name = tensor("op_679")]; + tensor var_680 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_680")]; + tensor window_11 = add(x = var_679, y = var_680)[name = tensor("window_11")]; + tensor var_682 = const()[name = tensor("op_682"), val = tensor(0x1p+0)]; + tensor var_683 = add(x = filled_5, y = var_682)[name = tensor("op_683")]; + tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_683)[name = tensor("clip_5")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_30, interleave = input_101_interleave_0, values = window_11)[name = tensor("input_101")]; tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; @@ -688,13 +695,13 @@ program(1.0) tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; - tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_38")]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; - tensor var_702_split_sizes_0 = const()[name = tensor("op_702_split_sizes_0"), val = tensor([256, 256])]; - tensor var_702_axis_0 = const()[name = tensor("op_702_axis_0"), val = tensor(1)]; - tensor var_702_0, tensor var_702_1 = split(axis = var_702_axis_0, split_sizes = var_702_split_sizes_0, x = inputs_23)[name = tensor("op_702")]; - tensor var_704 = sigmoid(x = var_702_1)[name = tensor("op_704")]; - tensor inputs_25 = mul(x = var_702_0, y = var_704)[name = tensor("inputs_25")]; + tensor var_706_split_sizes_0 = const()[name = tensor("op_706_split_sizes_0"), val = tensor([256, 256])]; + tensor var_706_axis_0 = const()[name = tensor("op_706_axis_0"), val = tensor(1)]; + tensor var_706_0, tensor var_706_1 = split(axis = var_706_axis_0, split_sizes = var_706_split_sizes_0, x = inputs_23)[name = tensor("op_706")]; + tensor var_708 = sigmoid(x = var_706_1)[name = tensor("op_708")]; + tensor inputs_25 = mul(x = var_706_0, y = var_708)[name = tensor("inputs_25")]; tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; @@ -714,22 +721,22 @@ program(1.0) tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_735_begin_0 = const()[name = tensor("op_735_begin_0"), val = tensor([0, -1, 0])]; - tensor var_735_end_0 = const()[name = tensor("op_735_end_0"), val = tensor([1, 16, 256])]; - tensor var_735_end_mask_0 = const()[name = tensor("op_735_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_37")]; - tensor var_735 = slice_by_index(begin = var_735_begin_0, end = var_735_end_0, end_mask = var_735_end_mask_0, x = conv_out_5)[name = tensor("op_735")]; - tensor var_738_interleave_0 = const()[name = tensor("op_738_interleave_0"), val = tensor(false)]; - tensor var_738 = concat(axis = var_33, interleave = var_738_interleave_0, values = var_735)[name = tensor("op_738")]; - tensor input_111 = add(x = x_15, y = var_738)[name = tensor("input_111")]; + tensor var_739_begin_0 = const()[name = tensor("op_739_begin_0"), val = tensor([0, -1, 0])]; + tensor var_739_end_0 = const()[name = tensor("op_739_end_0"), val = tensor([1, 16, 256])]; + tensor var_739_end_mask_0 = const()[name = tensor("op_739_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_739 = slice_by_index(begin = var_739_begin_0, end = var_739_end_0, end_mask = var_739_end_mask_0, x = conv_out_5)[name = tensor("op_739")]; + tensor var_741_perm_0 = const()[name = tensor("op_741_perm_0"), val = tensor([1, 0, 2])]; + tensor var_741 = transpose(perm = var_741_perm_0, x = var_739)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_741)[name = tensor("input_111")]; tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; - tensor var_761 = const()[name = tensor("op_761"), val = tensor(0x1p-1)]; - tensor var_762 = mul(x = input_119, y = var_761)[name = tensor("op_762")]; - tensor input_121 = add(x = var_762, y = input_111)[name = tensor("input_121")]; + tensor var_764 = const()[name = tensor("op_764"), val = tensor(0x1p-1)]; + tensor var_765 = mul(x = input_119, y = var_764)[name = tensor("op_765")]; + tensor input_121 = add(x = var_765, y = input_111)[name = tensor("input_121")]; tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; @@ -737,9 +744,9 @@ program(1.0) tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; - tensor var_791 = const()[name = tensor("op_791"), val = tensor(0x1p-1)]; - tensor var_792 = mul(x = input_131, y = var_791)[name = tensor("op_792")]; - tensor input_133 = add(x = var_792, y = input_123)[name = tensor("input_133")]; + tensor var_794 = const()[name = tensor("op_794"), val = tensor(0x1p-1)]; + tensor var_795 = mul(x = input_131, y = var_794)[name = tensor("op_795")]; + tensor input_133 = add(x = var_795, y = input_123)[name = tensor("input_133")]; tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; @@ -752,19 +759,19 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_806 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; - tensor var_807 = const()[name = tensor("op_807"), val = tensor([1, 1, 4, 64])]; - tensor var_808 = reshape(shape = var_807, x = var_806)[name = tensor("op_808")]; + tensor var_809 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_810 = const()[name = tensor("op_810"), val = tensor([1, 1, 4, 64])]; + tensor var_811 = reshape(shape = var_810, x = var_809)[name = tensor("op_811")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_812 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; - tensor var_813 = const()[name = tensor("op_813"), val = tensor(0x1p-3)]; - tensor var_814 = mul(x = var_812, y = var_813)[name = tensor("op_814")]; - tensor var_815 = const()[name = tensor("op_815"), val = tensor([1, 1, 4, 64])]; - tensor var_816 = reshape(shape = var_815, x = var_814)[name = tensor("op_816")]; + tensor var_815 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_816 = const()[name = tensor("op_816"), val = tensor(0x1p-3)]; + tensor var_817 = mul(x = var_815, y = var_816)[name = tensor("op_817")]; + tensor var_818 = const()[name = tensor("op_818"), val = tensor([1, 1, 4, 64])]; + tensor var_819 = reshape(shape = var_818, x = var_817)[name = tensor("op_819")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_820 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; - tensor var_821 = const()[name = tensor("op_821"), val = tensor([1, 1, 4, 64])]; - tensor var_822 = reshape(shape = var_821, x = var_820)[name = tensor("op_822")]; + tensor var_823 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_824 = const()[name = tensor("op_824"), val = tensor([1, 1, 4, 64])]; + tensor var_825 = reshape(shape = var_824, x = var_823)[name = tensor("op_825")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; @@ -772,84 +779,86 @@ program(1.0) tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_816)[name = tensor("transpose_35")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_808)[name = tensor("transpose_36")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_819)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_811)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_832 = const()[name = tensor("op_832"), val = tensor([1, 1])]; - tensor var_833 = reshape(shape = var_832, x = sqrt_s_t_7)[name = tensor("op_833")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_833)[name = tensor("M_7")]; - tensor var_835 = mul(x = qk_7, y = M_7)[name = tensor("op_835")]; + tensor var_835 = const()[name = tensor("op_835"), val = tensor([1, 1])]; + tensor var_836 = reshape(shape = var_835, x = sqrt_s_t_7)[name = tensor("op_836")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_836)[name = tensor("M_7")]; + tensor var_838 = mul(x = qk_7, y = M_7)[name = tensor("op_838")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_822)[name = tensor("transpose_34")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_835, y = v_7)[name = tensor("inner_7")]; - tensor var_837_transpose_x_0 = const()[name = tensor("op_837_transpose_x_0"), val = tensor(false)]; - tensor var_837_transpose_y_0 = const()[name = tensor("op_837_transpose_y_0"), val = tensor(false)]; - tensor var_837 = matmul(transpose_x = var_837_transpose_x_0, transpose_y = var_837_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_837")]; - tensor var_838 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_838")]; - tensor var_839 = const()[name = tensor("op_839"), val = tensor([1, 1, 1, 1])]; - tensor var_840 = reshape(shape = var_839, x = var_838)[name = tensor("op_840")]; - tensor cross_7 = mul(x = var_837, y = var_840)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_825)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_838, y = v_7)[name = tensor("inner_7")]; + tensor var_840_transpose_x_0 = const()[name = tensor("op_840_transpose_x_0"), val = tensor(false)]; + tensor var_840_transpose_y_0 = const()[name = tensor("op_840_transpose_y_0"), val = tensor(false)]; + tensor var_840 = matmul(transpose_x = var_840_transpose_x_0, transpose_y = var_840_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_840")]; + tensor var_841 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_841")]; + tensor var_842 = const()[name = tensor("op_842"), val = tensor([1, 1, 1, 1])]; + tensor var_843 = reshape(shape = var_842, x = var_841)[name = tensor("op_843")]; + tensor cross_7 = mul(x = var_840, y = var_843)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_843 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_843")]; - tensor var_845_transpose_x_1 = const()[name = tensor("op_845_transpose_x_1"), val = tensor(true)]; - tensor var_845_transpose_y_1 = const()[name = tensor("op_845_transpose_y_1"), val = tensor(false)]; - tensor var_845 = matmul(transpose_x = var_845_transpose_x_1, transpose_y = var_845_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_845")]; - tensor new_kv_unnorm_7 = add(x = var_843, y = var_845)[name = tensor("new_kv_unnorm_7")]; - tensor var_847 = const()[name = tensor("op_847"), val = tensor(0x1p+0)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_847)[name = tensor("new_scale_7")]; - tensor var_849 = sqrt(x = new_scale_7)[name = tensor("op_849")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_849)[name = tensor("nkv_1")]; - tensor var_851_perm_0 = const()[name = tensor("op_851_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_846 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_846")]; + tensor var_848_transpose_x_1 = const()[name = tensor("op_848_transpose_x_1"), val = tensor(true)]; + tensor var_848_transpose_y_1 = const()[name = tensor("op_848_transpose_y_1"), val = tensor(false)]; + tensor var_848 = matmul(transpose_x = var_848_transpose_x_1, transpose_y = var_848_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_848")]; + tensor new_kv_unnorm_7 = add(x = var_846, y = var_848)[name = tensor("new_kv_unnorm_7")]; + tensor var_850 = const()[name = tensor("op_850"), val = tensor(0x1p+0)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_850)[name = tensor("new_scale_7")]; + tensor var_852 = sqrt(x = new_scale_7)[name = tensor("op_852")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_852)[name = tensor("nkv_1")]; + tensor var_854_perm_0 = const()[name = tensor("op_854_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_851 = transpose(perm = var_851_perm_0, x = out_19)[name = tensor("transpose_33")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_851)[name = tensor("out_21")]; - tensor var_855 = const()[name = tensor("op_855"), val = tensor([1, 1, 256])]; - tensor out_23 = reshape(shape = var_855, x = out_21)[name = tensor("out_23")]; - tensor var_857 = silu(x = input_137)[name = tensor("op_857")]; - tensor input_139 = mul(x = var_857, y = out_23)[name = tensor("input_139")]; + tensor var_854 = transpose(perm = var_854_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_854)[name = tensor("out_21")]; + tensor var_858 = const()[name = tensor("op_858"), val = tensor([1, 1, 256])]; + tensor out_23 = reshape(shape = var_858, x = out_21)[name = tensor("out_23")]; + tensor var_860 = silu(x = input_137)[name = tensor("op_860")]; + tensor input_139 = mul(x = var_860, y = out_23)[name = tensor("input_139")]; tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor window_13_begin_0 = const()[name = tensor("window_13_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_13_end_0 = const()[name = tensor("window_13_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_13_end_mask_0 = const()[name = tensor("window_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_13_squeeze_mask_0 = const()[name = tensor("window_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_13 = slice_by_index(begin = window_13_begin_0, end = window_13_end_0, end_mask = window_13_end_mask_0, squeeze_mask = window_13_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_13")]; tensor filled_7_begin_0 = const()[name = tensor("filled_7_begin_0"), val = tensor([3, 0])]; tensor filled_7_end_0 = const()[name = tensor("filled_7_end_0"), val = tensor([4, 1])]; tensor filled_7_end_mask_0 = const()[name = tensor("filled_7_end_mask_0"), val = tensor([false, true])]; tensor filled_7_squeeze_mask_0 = const()[name = tensor("filled_7_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_7 = slice_by_index(begin = filled_7_begin_0, end = filled_7_end_0, end_mask = filled_7_end_mask_0, squeeze_mask = filled_7_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_7")]; - tensor var_869_begin_0 = const()[name = tensor("op_869_begin_0"), val = tensor([0, 1, 0])]; - tensor var_869_end_0 = const()[name = tensor("op_869_end_0"), val = tensor([1, 16, 256])]; - tensor var_869_end_mask_0 = const()[name = tensor("op_869_end_mask_0"), val = tensor([true, true, true])]; - tensor var_869 = slice_by_index(begin = var_869_begin_0, end = var_869_end_0, end_mask = var_869_end_mask_0, x = window)[name = tensor("op_869")]; + tensor var_872_begin_0 = const()[name = tensor("op_872_begin_0"), val = tensor([0, 1, 0])]; + tensor var_872_end_0 = const()[name = tensor("op_872_end_0"), val = tensor([1, 16, 256])]; + tensor var_872_end_mask_0 = const()[name = tensor("op_872_end_mask_0"), val = tensor([true, true, true])]; + tensor var_872 = slice_by_index(begin = var_872_begin_0, end = var_872_end_0, end_mask = var_872_end_mask_0, x = window_13)[name = tensor("op_872")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_869, x_21))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_872, x_21))[name = tensor("rolled")]; tensor clip_6 = clip(alpha = var_21, beta = var_20, x = filled_7)[name = tensor("clip_6")]; - tensor var_874 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_874")]; - tensor var_875_dtype_0 = const()[name = tensor("op_875_dtype_0"), val = tensor("fp32")]; - tensor var_876 = const()[name = tensor("op_876"), val = tensor([1, 16, 1])]; - tensor var_875 = cast(dtype = var_875_dtype_0, x = var_874)[name = tensor("cast_121")]; - tensor fill_mask = reshape(shape = var_876, x = var_875)[name = tensor("fill_mask")]; + tensor var_877 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_877")]; + tensor var_878_dtype_0 = const()[name = tensor("op_878_dtype_0"), val = tensor("fp32")]; + tensor var_879 = const()[name = tensor("op_879"), val = tensor([1, 16, 1])]; + tensor var_878 = cast(dtype = var_878_dtype_0, x = var_877)[name = tensor("cast_121")]; + tensor fill_mask = reshape(shape = var_879, x = var_878)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; tensor x_expanded = tile(reps = x_expanded_reps_0, x = x_21)[name = tensor("x_expanded")]; - tensor var_880 = sub(x = var_17, y = fill_mask)[name = tensor("op_880")]; - tensor var_881 = mul(x = window, y = var_880)[name = tensor("op_881")]; - tensor var_882 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_882")]; - tensor filled_window = add(x = var_881, y = var_882)[name = tensor("filled_window")]; - tensor var_884 = greater_equal(x = filled_7, y = var_16)[name = tensor("op_884")]; - tensor var_885_dtype_0 = const()[name = tensor("op_885_dtype_0"), val = tensor("fp32")]; - tensor var_886 = const()[name = tensor("op_886"), val = tensor([1, 1, 1])]; - tensor var_885 = cast(dtype = var_885_dtype_0, x = var_884)[name = tensor("cast_120")]; - tensor use_roll = reshape(shape = var_886, x = var_885)[name = tensor("use_roll")]; - tensor var_888 = sub(x = var_17, y = use_roll)[name = tensor("op_888")]; - tensor var_889 = mul(x = filled_window, y = var_888)[name = tensor("op_889")]; - tensor var_890 = mul(x = rolled, y = use_roll)[name = tensor("op_890")]; - tensor input_141 = add(x = var_889, y = var_890)[name = tensor("input_141")]; - tensor var_892 = const()[name = tensor("op_892"), val = tensor(0x1p+0)]; - tensor var_893 = add(x = filled_7, y = var_892)[name = tensor("op_893")]; - tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_893)[name = tensor("clip_7")]; + tensor var_883 = sub(x = var_17, y = fill_mask)[name = tensor("op_883")]; + tensor var_884 = mul(x = window_13, y = var_883)[name = tensor("op_884")]; + tensor var_885 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_885")]; + tensor filled_window = add(x = var_884, y = var_885)[name = tensor("filled_window")]; + tensor var_887 = greater_equal(x = filled_7, y = var_16)[name = tensor("op_887")]; + tensor var_888_dtype_0 = const()[name = tensor("op_888_dtype_0"), val = tensor("fp32")]; + tensor var_889 = const()[name = tensor("op_889"), val = tensor([1, 1, 1])]; + tensor var_888 = cast(dtype = var_888_dtype_0, x = var_887)[name = tensor("cast_120")]; + tensor use_roll = reshape(shape = var_889, x = var_888)[name = tensor("use_roll")]; + tensor var_891 = sub(x = var_17, y = use_roll)[name = tensor("op_891")]; + tensor var_892 = mul(x = filled_window, y = var_891)[name = tensor("op_892")]; + tensor var_893 = mul(x = rolled, y = use_roll)[name = tensor("op_893")]; + tensor window = add(x = var_892, y = var_893)[name = tensor("window")]; + tensor var_895 = const()[name = tensor("op_895"), val = tensor(0x1p+0)]; + tensor var_896 = add(x = filled_7, y = var_895)[name = tensor("op_896")]; + tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_896)[name = tensor("clip_7")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_30, interleave = input_141_interleave_0, values = window)[name = tensor("input_141")]; tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; @@ -858,13 +867,13 @@ program(1.0) tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; - tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_32")]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; - tensor var_914_split_sizes_0 = const()[name = tensor("op_914_split_sizes_0"), val = tensor([256, 256])]; - tensor var_914_axis_0 = const()[name = tensor("op_914_axis_0"), val = tensor(1)]; - tensor var_914_0, tensor var_914_1 = split(axis = var_914_axis_0, split_sizes = var_914_split_sizes_0, x = inputs_33)[name = tensor("op_914")]; - tensor var_916 = sigmoid(x = var_914_1)[name = tensor("op_916")]; - tensor inputs_35 = mul(x = var_914_0, y = var_916)[name = tensor("inputs_35")]; + tensor var_919_split_sizes_0 = const()[name = tensor("op_919_split_sizes_0"), val = tensor([256, 256])]; + tensor var_919_axis_0 = const()[name = tensor("op_919_axis_0"), val = tensor(1)]; + tensor var_919_0, tensor var_919_1 = split(axis = var_919_axis_0, split_sizes = var_919_split_sizes_0, x = inputs_33)[name = tensor("op_919")]; + tensor var_921 = sigmoid(x = var_919_1)[name = tensor("op_921")]; + tensor inputs_35 = mul(x = var_919_0, y = var_921)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; @@ -884,22 +893,22 @@ program(1.0) tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_947_begin_0 = const()[name = tensor("op_947_begin_0"), val = tensor([0, -1, 0])]; - tensor var_947_end_0 = const()[name = tensor("op_947_end_0"), val = tensor([1, 16, 256])]; - tensor var_947_end_mask_0 = const()[name = tensor("op_947_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_31")]; - tensor var_947 = slice_by_index(begin = var_947_begin_0, end = var_947_end_0, end_mask = var_947_end_mask_0, x = conv_out_7)[name = tensor("op_947")]; - tensor var_950_interleave_0 = const()[name = tensor("op_950_interleave_0"), val = tensor(false)]; - tensor var_950 = concat(axis = var_33, interleave = var_950_interleave_0, values = var_947)[name = tensor("op_950")]; - tensor input_151 = add(x = x_21, y = var_950)[name = tensor("input_151")]; + tensor var_952_begin_0 = const()[name = tensor("op_952_begin_0"), val = tensor([0, -1, 0])]; + tensor var_952_end_0 = const()[name = tensor("op_952_end_0"), val = tensor([1, 16, 256])]; + tensor var_952_end_mask_0 = const()[name = tensor("op_952_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_952 = slice_by_index(begin = var_952_begin_0, end = var_952_end_0, end_mask = var_952_end_mask_0, x = conv_out_7)[name = tensor("op_952")]; + tensor var_954_perm_0 = const()[name = tensor("op_954_perm_0"), val = tensor([1, 0, 2])]; + tensor var_954 = transpose(perm = var_954_perm_0, x = var_952)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_954)[name = tensor("input_151")]; tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; - tensor var_973 = const()[name = tensor("op_973"), val = tensor(0x1p-1)]; - tensor var_974 = mul(x = input_159, y = var_973)[name = tensor("op_974")]; - tensor input_161 = add(x = var_974, y = input_151)[name = tensor("input_161")]; + tensor var_977 = const()[name = tensor("op_977"), val = tensor(0x1p-1)]; + tensor var_978 = mul(x = input_159, y = var_977)[name = tensor("op_978")]; + tensor input_161 = add(x = var_978, y = input_151)[name = tensor("input_161")]; tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; @@ -912,44 +921,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_992_begin_0 = const()[name = tensor("op_992_begin_0"), val = tensor([0, 0, 1])]; - tensor var_992_end_0 = const()[name = tensor("op_992_end_0"), val = tensor([1, 256, 19])]; - tensor var_992_end_mask_0 = const()[name = tensor("op_992_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_992_begin_0, end = var_992_end_0, end_mask = var_992_end_mask_0, x = cat)[name = tensor("op_992")]; + tensor var_996_begin_0 = const()[name = tensor("op_996_begin_0"), val = tensor([0, 0, 1])]; + tensor var_996_end_0 = const()[name = tensor("op_996_end_0"), val = tensor([1, 256, 19])]; + tensor var_996_end_mask_0 = const()[name = tensor("op_996_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_996_begin_0, end = var_996_end_0, end_mask = var_996_end_mask_0, x = cat)[name = tensor("op_996")]; tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; - tensor var_994 = const()[name = tensor("op_994"), val = tensor([-1])]; + tensor var_998 = const()[name = tensor("op_998"), val = tensor([-1])]; tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_995 = reduce_l2_norm(axes = var_994, keep_dims = var_36, x = input_163)[name = tensor("op_995")]; + tensor var_999 = reduce_l2_norm(axes = var_998, keep_dims = var_36, x = input_163)[name = tensor("op_999")]; tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; - tensor clip_8 = clip(alpha = var_12, beta = const_12, x = var_995)[name = tensor("clip_8")]; + tensor clip_8 = clip(alpha = var_12, beta = const_12, x = var_999)[name = tensor("clip_8")]; tensor emb = real_div(x = input_163, y = clip_8)[name = tensor("emb")]; - tensor var_999_axis_0 = const()[name = tensor("op_999_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_999_axis_0, values = (var_214, var_426, var_638, nkv_1))[name = tensor("op_999")]; - tensor var_1001_axis_0 = const()[name = tensor("op_1001_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_1001_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1001")]; tensor var_1003_axis_0 = const()[name = tensor("op_1003_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_1003_axis_0, values = (input_21, input_61, input_101, input_141))[name = tensor("op_1003")]; + tensor enc_kv_new = stack(axis = var_1003_axis_0, values = (var_214, var_427, var_640, nkv_1))[name = tensor("op_1003")]; tensor var_1005_axis_0 = const()[name = tensor("op_1005_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_1005_axis_0, values = (clip_1, clip_3, clip_5, clip_7))[name = tensor("op_1005")]; - tensor var_1015 = const()[name = tensor("op_1015"), val = tensor(0x1.5798eep-27)]; - tensor var_1021 = const()[name = tensor("op_1021"), val = tensor(0x1.4f8b58p-17)]; - tensor var_1023 = const()[name = tensor("op_1023"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_1024 = const()[name = tensor("op_1024"), val = tensor(true)]; - tensor var_1026 = const()[name = tensor("op_1026"), val = tensor(0x1p+0)]; - tensor var_1030 = const()[name = tensor("op_1030"), val = tensor(-1)]; - tensor var_1035 = const()[name = tensor("op_1035"), val = tensor(0)]; - tensor var_1092 = const()[name = tensor("op_1092"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395456)))]; - tensor var_1097_axes_0 = const()[name = tensor("op_1097_axes_0"), val = tensor([2])]; - tensor var_1097 = expand_dims(axes = var_1097_axes_0, x = emb)[name = tensor("op_1097")]; - tensor var_1098 = const()[name = tensor("op_1098"), val = tensor([1, 1, 12, 1])]; - tensor emb_exp = tile(reps = var_1098, x = var_1097)[name = tensor("emb_exp")]; + tensor enc_scale_new = stack(axis = var_1005_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1005")]; + tensor var_1007_axis_0 = const()[name = tensor("op_1007_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1007_axis_0, values = (window_3, window_7, window_11, window))[name = tensor("op_1007")]; + tensor var_1009_axis_0 = const()[name = tensor("op_1009_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1009_axis_0, values = (clip_1, clip_3, clip_5, clip_7))[name = tensor("op_1009")]; + tensor var_1019 = const()[name = tensor("op_1019"), val = tensor(0x1.5798eep-27)]; + tensor var_1024 = const()[name = tensor("op_1024"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1026 = const()[name = tensor("op_1026"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1027 = const()[name = tensor("op_1027"), val = tensor(true)]; + tensor var_1029 = const()[name = tensor("op_1029"), val = tensor(0x1p+0)]; + tensor var_1033 = const()[name = tensor("op_1033"), val = tensor(-1)]; + tensor var_1039 = const()[name = tensor("op_1039"), val = tensor(0)]; + tensor var_1096 = const()[name = tensor("op_1096"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395456)))]; + tensor var_1101_axes_0 = const()[name = tensor("op_1101_axes_0"), val = tensor([2])]; + tensor var_1101 = expand_dims(axes = var_1101_axes_0, x = emb)[name = tensor("op_1101")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 12, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1101)[name = tensor("emb_exp")]; tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; - tensor input_165 = concat(axis = var_1030, interleave = input_165_interleave_0, values = (emb_exp, var_1092))[name = tensor("input_165")]; + tensor input_165 = concat(axis = var_1033, interleave = input_165_interleave_0, values = (emb_exp, var_1096))[name = tensor("input_165")]; tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; - tensor var_1105_perm_0 = const()[name = tensor("op_1105_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1109 = const()[name = tensor("op_1109"), val = tensor([12, 1, 256])]; - tensor var_1105 = transpose(perm = var_1105_perm_0, x = x_27)[name = tensor("transpose_28")]; - tensor x_29 = reshape(shape = var_1109, x = var_1105)[name = tensor("x_29")]; + tensor var_1109_perm_0 = const()[name = tensor("op_1109_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1113 = const()[name = tensor("op_1113"), val = tensor([12, 1, 256])]; + tensor var_1109 = transpose(perm = var_1109_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1113, x = var_1109)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 12, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -960,131 +969,131 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_1117 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; - tensor var_1118 = const()[name = tensor("op_1118"), val = tensor([12, 1, 4, 64])]; - tensor var_1119 = reshape(shape = var_1118, x = var_1117)[name = tensor("op_1119")]; + tensor var_1121 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1122 = const()[name = tensor("op_1122"), val = tensor([12, 1, 4, 64])]; + tensor var_1123 = reshape(shape = var_1122, x = var_1121)[name = tensor("op_1123")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1123 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; - tensor var_1124 = const()[name = tensor("op_1124"), val = tensor(0x1p-3)]; - tensor var_1125 = mul(x = var_1123, y = var_1124)[name = tensor("op_1125")]; - tensor var_1126 = const()[name = tensor("op_1126"), val = tensor([12, 1, 4, 64])]; - tensor var_1127 = reshape(shape = var_1126, x = var_1125)[name = tensor("op_1127")]; + tensor var_1127 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1128 = const()[name = tensor("op_1128"), val = tensor(0x1p-3)]; + tensor var_1129 = mul(x = var_1127, y = var_1128)[name = tensor("op_1129")]; + tensor var_1130 = const()[name = tensor("op_1130"), val = tensor([12, 1, 4, 64])]; + tensor var_1131 = reshape(shape = var_1130, x = var_1129)[name = tensor("op_1131")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1131 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; - tensor var_1132 = const()[name = tensor("op_1132"), val = tensor([12, 1, 4, 64])]; - tensor var_1133 = reshape(shape = var_1132, x = var_1131)[name = tensor("op_1133")]; + tensor var_1135 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1136 = const()[name = tensor("op_1136"), val = tensor([12, 1, 4, 64])]; + tensor var_1137 = reshape(shape = var_1136, x = var_1135)[name = tensor("op_1137")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_1035, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1039, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; - tensor clip_9 = clip(alpha = var_1026, beta = const_20, x = s_t_9)[name = tensor("clip_9")]; + tensor clip_9 = clip(alpha = var_1029, beta = const_20, x = s_t_9)[name = tensor("clip_9")]; tensor sqrt_s_t_9 = sqrt(x = clip_9)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_1127)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_1119)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1131)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1123)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_1145 = const()[name = tensor("op_1145"), val = tensor([1, 1])]; - tensor var_1146 = reshape(shape = var_1145, x = valid_mask)[name = tensor("op_1146")]; - tensor var_1148 = const()[name = tensor("op_1148"), val = tensor([1, 1])]; - tensor var_1149 = reshape(shape = var_1148, x = sqrt_s_t_9)[name = tensor("op_1149")]; - tensor M_9 = real_div(x = var_1146, y = var_1149)[name = tensor("M_9")]; - tensor var_1151 = mul(x = qk_9, y = M_9)[name = tensor("op_1151")]; + tensor var_1149 = const()[name = tensor("op_1149"), val = tensor([1, 1])]; + tensor var_1150 = reshape(shape = var_1149, x = valid_mask)[name = tensor("op_1150")]; + tensor var_1152 = const()[name = tensor("op_1152"), val = tensor([1, 1])]; + tensor var_1153 = reshape(shape = var_1152, x = sqrt_s_t_9)[name = tensor("op_1153")]; + tensor M_9 = real_div(x = var_1150, y = var_1153)[name = tensor("M_9")]; + tensor var_1155 = mul(x = qk_9, y = M_9)[name = tensor("op_1155")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_1133)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1151, y = v_9)[name = tensor("inner_9")]; - tensor var_1153_transpose_x_0 = const()[name = tensor("op_1153_transpose_x_0"), val = tensor(false)]; - tensor var_1153_transpose_y_0 = const()[name = tensor("op_1153_transpose_y_0"), val = tensor(false)]; - tensor var_1153 = matmul(transpose_x = var_1153_transpose_x_0, transpose_y = var_1153_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1153")]; - tensor var_1154 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1154")]; - tensor var_1155 = const()[name = tensor("op_1155"), val = tensor([1, 1, 1, 1])]; - tensor var_1156 = reshape(shape = var_1155, x = var_1154)[name = tensor("op_1156")]; - tensor cross_9 = mul(x = var_1153, y = var_1156)[name = tensor("cross_9")]; - tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1137)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1155, y = v_9)[name = tensor("inner_9")]; + tensor var_1157_transpose_x_0 = const()[name = tensor("op_1157_transpose_x_0"), val = tensor(false)]; + tensor var_1157_transpose_y_0 = const()[name = tensor("op_1157_transpose_y_0"), val = tensor(false)]; + tensor var_1157 = matmul(transpose_x = var_1157_transpose_x_0, transpose_y = var_1157_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1157")]; + tensor var_1158 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1158")]; tensor var_1159 = const()[name = tensor("op_1159"), val = tensor([1, 1, 1, 1])]; - tensor var_1160 = reshape(shape = var_1159, x = valid_mask)[name = tensor("op_1160")]; - tensor v_masked_1 = mul(x = v_9, y = var_1160)[name = tensor("v_masked_1")]; - tensor var_1162 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1162")]; - tensor var_1164_transpose_x_1 = const()[name = tensor("op_1164_transpose_x_1"), val = tensor(true)]; - tensor var_1164_transpose_y_1 = const()[name = tensor("op_1164_transpose_y_1"), val = tensor(false)]; - tensor var_1164 = matmul(transpose_x = var_1164_transpose_x_1, transpose_y = var_1164_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1164")]; - tensor new_kv_unnorm_9 = add(x = var_1162, y = var_1164)[name = tensor("new_kv_unnorm_9")]; - tensor var_1166_keep_dims_0 = const()[name = tensor("op_1166_keep_dims_0"), val = tensor(false)]; - tensor var_1166 = reduce_sum(keep_dims = var_1166_keep_dims_0, x = valid_mask)[name = tensor("op_1166")]; - tensor var_1167 = const()[name = tensor("op_1167"), val = tensor([1])]; - tensor var_1168 = reshape(shape = var_1167, x = var_1166)[name = tensor("op_1168")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_1168)[name = tensor("new_scale_9")]; + tensor var_1160 = reshape(shape = var_1159, x = var_1158)[name = tensor("op_1160")]; + tensor cross_9 = mul(x = var_1157, y = var_1160)[name = tensor("cross_9")]; + tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; + tensor var_1163 = const()[name = tensor("op_1163"), val = tensor([1, 1, 1, 1])]; + tensor var_1164 = reshape(shape = var_1163, x = valid_mask)[name = tensor("op_1164")]; + tensor v_masked_1 = mul(x = v_9, y = var_1164)[name = tensor("v_masked_1")]; + tensor var_1166 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1166")]; + tensor var_1168_transpose_x_1 = const()[name = tensor("op_1168_transpose_x_1"), val = tensor(true)]; + tensor var_1168_transpose_y_1 = const()[name = tensor("op_1168_transpose_y_1"), val = tensor(false)]; + tensor var_1168 = matmul(transpose_x = var_1168_transpose_x_1, transpose_y = var_1168_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1168")]; + tensor new_kv_unnorm_9 = add(x = var_1166, y = var_1168)[name = tensor("new_kv_unnorm_9")]; + tensor var_1170_keep_dims_0 = const()[name = tensor("op_1170_keep_dims_0"), val = tensor(false)]; + tensor var_1170 = reduce_sum(keep_dims = var_1170_keep_dims_0, x = valid_mask)[name = tensor("op_1170")]; + tensor var_1171 = const()[name = tensor("op_1171"), val = tensor([1])]; + tensor var_1172 = reshape(shape = var_1171, x = var_1170)[name = tensor("op_1172")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1172)[name = tensor("new_scale_9")]; tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; - tensor clip_10 = clip(alpha = var_1026, beta = const_21, x = new_scale_9)[name = tensor("clip_10")]; + tensor clip_10 = clip(alpha = var_1029, beta = const_21, x = new_scale_9)[name = tensor("clip_10")]; tensor sqrt_new_scale_1 = sqrt(x = clip_10)[name = tensor("sqrt_new_scale_1")]; - tensor var_1172 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1172")]; - tensor var_1173_perm_0 = const()[name = tensor("op_1173_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1176 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1176")]; + tensor var_1177_perm_0 = const()[name = tensor("op_1177_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_1173 = transpose(perm = var_1173_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1023, x = var_1173)[name = tensor("out_27")]; - tensor var_1177 = const()[name = tensor("op_1177"), val = tensor([12, 1, 256])]; - tensor out_29 = reshape(shape = var_1177, x = out_27)[name = tensor("out_29")]; - tensor var_1179 = silu(x = input_169)[name = tensor("op_1179")]; - tensor input_171 = mul(x = var_1179, y = out_29)[name = tensor("input_171")]; + tensor var_1177 = transpose(perm = var_1177_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1026, x = var_1177)[name = tensor("out_27")]; + tensor var_1181 = const()[name = tensor("op_1181"), val = tensor([12, 1, 256])]; + tensor out_29 = reshape(shape = var_1181, x = out_27)[name = tensor("out_29")]; + tensor var_1183 = silu(x = input_169)[name = tensor("op_1183")]; + tensor input_171 = mul(x = var_1183, y = out_29)[name = tensor("input_171")]; tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1021, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; - tensor var_1189 = const()[name = tensor("op_1189"), val = tensor([1, 12, 1, 256])]; - tensor var_1190 = reshape(shape = var_1189, x = xt_1)[name = tensor("op_1190")]; - tensor var_1191_perm_0 = const()[name = tensor("op_1191_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1194 = const()[name = tensor("op_1194"), val = tensor([1, 12, 256])]; - tensor var_1191 = transpose(perm = var_1191_perm_0, x = var_1190)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_1194, x = var_1191)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1024, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1193 = const()[name = tensor("op_1193"), val = tensor([1, 12, 1, 256])]; + tensor var_1194 = reshape(shape = var_1193, x = xt_1)[name = tensor("op_1194")]; + tensor var_1195_perm_0 = const()[name = tensor("op_1195_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1198 = const()[name = tensor("op_1198"), val = tensor([1, 12, 256])]; + tensor var_1195 = transpose(perm = var_1195_perm_0, x = var_1194)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1198, x = var_1195)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_1217 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_4 = const()[name = tensor("concat_4"), val = tensor([12, 1, 3, 256])]; - tensor var_1219 = reshape(shape = concat_4, x = var_1217)[name = tensor("op_1219")]; - tensor var_1220_axes_0 = const()[name = tensor("op_1220_axes_0"), val = tensor([0])]; - tensor var_1220 = expand_dims(axes = var_1220_axes_0, x = var_1219)[name = tensor("op_1220")]; - tensor var_1221_perm_0 = const()[name = tensor("op_1221_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1222_axes_0 = const()[name = tensor("op_1222_axes_0"), val = tensor([-2])]; - tensor var_1221 = transpose(perm = var_1221_perm_0, x = var_1220)[name = tensor("transpose_21")]; - tensor var_1222 = squeeze(axes = var_1222_axes_0, x = var_1221)[name = tensor("op_1222")]; + tensor var_1221 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_5 = const()[name = tensor("concat_5"), val = tensor([12, 1, 3, 256])]; + tensor var_1223 = reshape(shape = concat_5, x = var_1221)[name = tensor("op_1223")]; + tensor var_1224_axes_0 = const()[name = tensor("op_1224_axes_0"), val = tensor([0])]; + tensor var_1224 = expand_dims(axes = var_1224_axes_0, x = var_1223)[name = tensor("op_1224")]; + tensor var_1225_perm_0 = const()[name = tensor("op_1225_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1226_axes_0 = const()[name = tensor("op_1226_axes_0"), val = tensor([-2])]; + tensor var_1225 = transpose(perm = var_1225_perm_0, x = var_1224)[name = tensor("transpose_21")]; + tensor var_1226 = squeeze(axes = var_1226_axes_0, x = var_1225)[name = tensor("op_1226")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 12, 1, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1222)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1226)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 12, 1, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1222)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1226)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 12, 1, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1222)[name = tensor("v_11")]; - tensor var_1230 = const()[name = tensor("op_1230"), val = tensor([12, 4, 64])]; - tensor var_1231 = reshape(shape = var_1230, x = q_11)[name = tensor("op_1231")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1226)[name = tensor("v_11")]; + tensor var_1234 = const()[name = tensor("op_1234"), val = tensor([12, 4, 64])]; + tensor var_1235 = reshape(shape = var_1234, x = q_11)[name = tensor("op_1235")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1237 = const()[name = tensor("op_1237"), val = tensor([12, 4, 64])]; - tensor var_1238 = reshape(shape = var_1237, x = k_11)[name = tensor("op_1238")]; + tensor var_1241 = const()[name = tensor("op_1241"), val = tensor([12, 4, 64])]; + tensor var_1242 = reshape(shape = var_1241, x = k_11)[name = tensor("op_1242")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1244 = const()[name = tensor("op_1244"), val = tensor([12, 4, 64])]; - tensor var_1245 = reshape(shape = var_1244, x = v_11)[name = tensor("op_1245")]; + tensor var_1248 = const()[name = tensor("op_1248"), val = tensor([12, 4, 64])]; + tensor var_1249 = reshape(shape = var_1248, x = v_11)[name = tensor("op_1249")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1248 = const()[name = tensor("op_1248"), val = tensor([1, 4, 12, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_1231)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_1248, x = q_13)[name = tensor("q_15")]; - tensor var_1250 = const()[name = tensor("op_1250"), val = tensor([1, 4, 12, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_1238)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_1250, x = k_13)[name = tensor("k_15")]; tensor var_1252 = const()[name = tensor("op_1252"), val = tensor([1, 4, 12, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_1245)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_1252, x = v_13)[name = tensor("v_15")]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1235)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1252, x = q_13)[name = tensor("q_15")]; + tensor var_1254 = const()[name = tensor("op_1254"), val = tensor([1, 4, 12, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1242)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1254, x = k_13)[name = tensor("k_15")]; + tensor var_1256 = const()[name = tensor("op_1256"), val = tensor([1, 4, 12, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1249)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1256, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -1095,30 +1104,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_1255 = const()[name = tensor("op_1255"), val = tensor([2, 0, 1, 3])]; - tensor var_1260 = const()[name = tensor("op_1260"), val = tensor([12, 256])]; - tensor var_1256 = transpose(perm = var_1255, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_1260, x = var_1256)[name = tensor("attn_output_3")]; + tensor var_1259 = const()[name = tensor("op_1259"), val = tensor([2, 0, 1, 3])]; + tensor var_1264 = const()[name = tensor("op_1264"), val = tensor([12, 256])]; + tensor var_1260 = transpose(perm = var_1259, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1264, x = var_1260)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_1264 = const()[name = tensor("op_1264"), val = tensor([12, 1, 256])]; - tensor attn_output_7 = reshape(shape = var_1264, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1268 = const()[name = tensor("op_1268"), val = tensor([12, 1, 256])]; + tensor attn_output_7 = reshape(shape = var_1268, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; - tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1021, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1024, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1021, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; - tensor var_1284 = const()[name = tensor("op_1284"), val = tensor([1, 1, 12, 256])]; - tensor x_31 = reshape(shape = var_1284, x = xt_3)[name = tensor("x_31")]; - tensor var_1286_perm_0 = const()[name = tensor("op_1286_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1290 = const()[name = tensor("op_1290"), val = tensor([12, 1, 256])]; - tensor var_1286 = transpose(perm = var_1286_perm_0, x = x_31)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_1290, x = var_1286)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1024, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1288 = const()[name = tensor("op_1288"), val = tensor([1, 1, 12, 256])]; + tensor x_31 = reshape(shape = var_1288, x = xt_3)[name = tensor("x_31")]; + tensor var_1290_perm_0 = const()[name = tensor("op_1290_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1294 = const()[name = tensor("op_1294"), val = tensor([12, 1, 256])]; + tensor var_1290 = transpose(perm = var_1290_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1294, x = var_1290)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 12, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1129,120 +1138,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_1298 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_1299 = const()[name = tensor("op_1299"), val = tensor([12, 1, 4, 64])]; - tensor var_1300 = reshape(shape = var_1299, x = var_1298)[name = tensor("op_1300")]; + tensor var_1302 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1303 = const()[name = tensor("op_1303"), val = tensor([12, 1, 4, 64])]; + tensor var_1304 = reshape(shape = var_1303, x = var_1302)[name = tensor("op_1304")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1304 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_1305 = const()[name = tensor("op_1305"), val = tensor(0x1p-3)]; - tensor var_1306 = mul(x = var_1304, y = var_1305)[name = tensor("op_1306")]; - tensor var_1307 = const()[name = tensor("op_1307"), val = tensor([12, 1, 4, 64])]; - tensor var_1308 = reshape(shape = var_1307, x = var_1306)[name = tensor("op_1308")]; + tensor var_1308 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1309 = const()[name = tensor("op_1309"), val = tensor(0x1p-3)]; + tensor var_1310 = mul(x = var_1308, y = var_1309)[name = tensor("op_1310")]; + tensor var_1311 = const()[name = tensor("op_1311"), val = tensor([12, 1, 4, 64])]; + tensor var_1312 = reshape(shape = var_1311, x = var_1310)[name = tensor("op_1312")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1312 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_1313 = const()[name = tensor("op_1313"), val = tensor([12, 1, 4, 64])]; - tensor var_1314 = reshape(shape = var_1313, x = var_1312)[name = tensor("op_1314")]; + tensor var_1316 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1317 = const()[name = tensor("op_1317"), val = tensor([12, 1, 4, 64])]; + tensor var_1318 = reshape(shape = var_1317, x = var_1316)[name = tensor("op_1318")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; - tensor clip_11 = clip(alpha = var_1026, beta = const_32, x = s_t)[name = tensor("clip_11")]; + tensor clip_11 = clip(alpha = var_1029, beta = const_32, x = s_t)[name = tensor("clip_11")]; tensor sqrt_s_t = sqrt(x = clip_11)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_1308)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_1300)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1312)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1304)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_1329 = const()[name = tensor("op_1329"), val = tensor([1, 1])]; - tensor var_1330 = reshape(shape = var_1329, x = sqrt_s_t)[name = tensor("op_1330")]; - tensor M = real_div(x = var_1146, y = var_1330)[name = tensor("M")]; - tensor var_1332 = mul(x = qk, y = M)[name = tensor("op_1332")]; + tensor var_1333 = const()[name = tensor("op_1333"), val = tensor([1, 1])]; + tensor var_1334 = reshape(shape = var_1333, x = sqrt_s_t)[name = tensor("op_1334")]; + tensor M = real_div(x = var_1150, y = var_1334)[name = tensor("M")]; + tensor var_1336 = mul(x = qk, y = M)[name = tensor("op_1336")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_1314)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1332, y = v_17)[name = tensor("inner")]; - tensor var_1334_transpose_x_0 = const()[name = tensor("op_1334_transpose_x_0"), val = tensor(false)]; - tensor var_1334_transpose_y_0 = const()[name = tensor("op_1334_transpose_y_0"), val = tensor(false)]; - tensor var_1334 = matmul(transpose_x = var_1334_transpose_x_0, transpose_y = var_1334_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1334")]; - tensor var_1335 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1335")]; - tensor var_1336 = const()[name = tensor("op_1336"), val = tensor([1, 1, 1, 1])]; - tensor var_1337 = reshape(shape = var_1336, x = var_1335)[name = tensor("op_1337")]; - tensor cross = mul(x = var_1334, y = var_1337)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1318)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1336, y = v_17)[name = tensor("inner")]; + tensor var_1338_transpose_x_0 = const()[name = tensor("op_1338_transpose_x_0"), val = tensor(false)]; + tensor var_1338_transpose_y_0 = const()[name = tensor("op_1338_transpose_y_0"), val = tensor(false)]; + tensor var_1338 = matmul(transpose_x = var_1338_transpose_x_0, transpose_y = var_1338_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1338")]; + tensor var_1339 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1339")]; + tensor var_1340 = const()[name = tensor("op_1340"), val = tensor([1, 1, 1, 1])]; + tensor var_1341 = reshape(shape = var_1340, x = var_1339)[name = tensor("op_1341")]; + tensor cross = mul(x = var_1338, y = var_1341)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_1160)[name = tensor("v_masked")]; - tensor var_1343 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1343")]; - tensor var_1345_transpose_x_1 = const()[name = tensor("op_1345_transpose_x_1"), val = tensor(true)]; - tensor var_1345_transpose_y_1 = const()[name = tensor("op_1345_transpose_y_1"), val = tensor(false)]; - tensor var_1345 = matmul(transpose_x = var_1345_transpose_x_1, transpose_y = var_1345_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1345")]; - tensor new_kv_unnorm = add(x = var_1343, y = var_1345)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_1168)[name = tensor("new_scale")]; + tensor v_masked = mul(x = v_17, y = var_1164)[name = tensor("v_masked")]; + tensor var_1347 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1347")]; + tensor var_1349_transpose_x_1 = const()[name = tensor("op_1349_transpose_x_1"), val = tensor(true)]; + tensor var_1349_transpose_y_1 = const()[name = tensor("op_1349_transpose_y_1"), val = tensor(false)]; + tensor var_1349 = matmul(transpose_x = var_1349_transpose_x_1, transpose_y = var_1349_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1349")]; + tensor new_kv_unnorm = add(x = var_1347, y = var_1349)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1172)[name = tensor("new_scale")]; tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; - tensor clip_12 = clip(alpha = var_1026, beta = const_33, x = new_scale)[name = tensor("clip_12")]; + tensor clip_12 = clip(alpha = var_1029, beta = const_33, x = new_scale)[name = tensor("clip_12")]; tensor sqrt_new_scale = sqrt(x = clip_12)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_1354_perm_0 = const()[name = tensor("op_1354_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1358_perm_0 = const()[name = tensor("op_1358_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_1354 = transpose(perm = var_1354_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1023, x = var_1354)[name = tensor("out_33")]; - tensor var_1358 = const()[name = tensor("op_1358"), val = tensor([12, 1, 256])]; - tensor out = reshape(shape = var_1358, x = out_33)[name = tensor("out")]; - tensor var_1360 = silu(x = input_187)[name = tensor("op_1360")]; - tensor input_189 = mul(x = var_1360, y = out)[name = tensor("input_189")]; + tensor var_1358 = transpose(perm = var_1358_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1026, x = var_1358)[name = tensor("out_33")]; + tensor var_1362 = const()[name = tensor("op_1362"), val = tensor([12, 1, 256])]; + tensor out = reshape(shape = var_1362, x = out_33)[name = tensor("out")]; + tensor var_1364 = silu(x = input_187)[name = tensor("op_1364")]; + tensor input_189 = mul(x = var_1364, y = out)[name = tensor("input_189")]; tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1021, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; - tensor var_1370 = const()[name = tensor("op_1370"), val = tensor([1, 12, 1, 256])]; - tensor var_1371 = reshape(shape = var_1370, x = xt_5)[name = tensor("op_1371")]; - tensor var_1372_perm_0 = const()[name = tensor("op_1372_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1375 = const()[name = tensor("op_1375"), val = tensor([1, 12, 256])]; - tensor var_1372 = transpose(perm = var_1372_perm_0, x = var_1371)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_1375, x = var_1372)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1024, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1374 = const()[name = tensor("op_1374"), val = tensor([1, 12, 1, 256])]; + tensor var_1375 = reshape(shape = var_1374, x = xt_5)[name = tensor("op_1375")]; + tensor var_1376_perm_0 = const()[name = tensor("op_1376_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1379 = const()[name = tensor("op_1379"), val = tensor([1, 12, 256])]; + tensor var_1376 = transpose(perm = var_1376_perm_0, x = var_1375)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1379, x = var_1376)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_1398 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_5 = const()[name = tensor("concat_5"), val = tensor([12, 1, 3, 256])]; - tensor var_1400 = reshape(shape = concat_5, x = var_1398)[name = tensor("op_1400")]; - tensor var_1401_axes_0 = const()[name = tensor("op_1401_axes_0"), val = tensor([0])]; - tensor var_1401 = expand_dims(axes = var_1401_axes_0, x = var_1400)[name = tensor("op_1401")]; - tensor var_1402_perm_0 = const()[name = tensor("op_1402_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1403_axes_0 = const()[name = tensor("op_1403_axes_0"), val = tensor([-2])]; - tensor var_1402 = transpose(perm = var_1402_perm_0, x = var_1401)[name = tensor("transpose_8")]; - tensor var_1403 = squeeze(axes = var_1403_axes_0, x = var_1402)[name = tensor("op_1403")]; + tensor var_1402 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_6 = const()[name = tensor("concat_6"), val = tensor([12, 1, 3, 256])]; + tensor var_1404 = reshape(shape = concat_6, x = var_1402)[name = tensor("op_1404")]; + tensor var_1405_axes_0 = const()[name = tensor("op_1405_axes_0"), val = tensor([0])]; + tensor var_1405 = expand_dims(axes = var_1405_axes_0, x = var_1404)[name = tensor("op_1405")]; + tensor var_1406_perm_0 = const()[name = tensor("op_1406_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1407_axes_0 = const()[name = tensor("op_1407_axes_0"), val = tensor([-2])]; + tensor var_1406 = transpose(perm = var_1406_perm_0, x = var_1405)[name = tensor("transpose_8")]; + tensor var_1407 = squeeze(axes = var_1407_axes_0, x = var_1406)[name = tensor("op_1407")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 12, 1, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1403)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1407)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 12, 1, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1403)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1407)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 12, 1, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1403)[name = tensor("v_19")]; - tensor var_1411 = const()[name = tensor("op_1411"), val = tensor([12, 4, 64])]; - tensor var_1412 = reshape(shape = var_1411, x = q_19)[name = tensor("op_1412")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1407)[name = tensor("v_19")]; + tensor var_1415 = const()[name = tensor("op_1415"), val = tensor([12, 4, 64])]; + tensor var_1416 = reshape(shape = var_1415, x = q_19)[name = tensor("op_1416")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1418 = const()[name = tensor("op_1418"), val = tensor([12, 4, 64])]; - tensor var_1419 = reshape(shape = var_1418, x = k_19)[name = tensor("op_1419")]; + tensor var_1422 = const()[name = tensor("op_1422"), val = tensor([12, 4, 64])]; + tensor var_1423 = reshape(shape = var_1422, x = k_19)[name = tensor("op_1423")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1425 = const()[name = tensor("op_1425"), val = tensor([12, 4, 64])]; - tensor var_1426 = reshape(shape = var_1425, x = v_19)[name = tensor("op_1426")]; + tensor var_1429 = const()[name = tensor("op_1429"), val = tensor([12, 4, 64])]; + tensor var_1430 = reshape(shape = var_1429, x = v_19)[name = tensor("op_1430")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1429 = const()[name = tensor("op_1429"), val = tensor([1, 4, 12, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_1412)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_1429, x = q_21)[name = tensor("q")]; - tensor var_1431 = const()[name = tensor("op_1431"), val = tensor([1, 4, 12, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_1419)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_1431, x = k_21)[name = tensor("k")]; tensor var_1433 = const()[name = tensor("op_1433"), val = tensor([1, 4, 12, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_1426)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_1433, x = v_21)[name = tensor("v")]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1416)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1433, x = q_21)[name = tensor("q")]; + tensor var_1435 = const()[name = tensor("op_1435"), val = tensor([1, 4, 12, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1423)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1435, x = k_21)[name = tensor("k")]; + tensor var_1437 = const()[name = tensor("op_1437"), val = tensor([1, 4, 12, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1430)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1437, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -1253,35 +1262,35 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_1436 = const()[name = tensor("op_1436"), val = tensor([2, 0, 1, 3])]; - tensor var_1441 = const()[name = tensor("op_1441"), val = tensor([12, 256])]; - tensor var_1437 = transpose(perm = var_1436, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_1441, x = var_1437)[name = tensor("attn_output_11")]; + tensor var_1440 = const()[name = tensor("op_1440"), val = tensor([2, 0, 1, 3])]; + tensor var_1445 = const()[name = tensor("op_1445"), val = tensor([12, 256])]; + tensor var_1441 = transpose(perm = var_1440, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1445, x = var_1441)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_1445 = const()[name = tensor("op_1445"), val = tensor([12, 1, 256])]; - tensor attn_output = reshape(shape = var_1445, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1449 = const()[name = tensor("op_1449"), val = tensor([12, 1, 256])]; + tensor attn_output = reshape(shape = var_1449, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; - tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1021, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1024, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1021, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; - tensor var_1465 = const()[name = tensor("op_1465"), val = tensor([1, 1, 12, 256])]; - tensor input = reshape(shape = var_1465, x = xt)[name = tensor("input")]; - tensor var_1467 = const()[name = tensor("op_1467"), val = tensor([-1])]; - tensor var_1468 = reduce_l2_norm(axes = var_1467, keep_dims = var_1024, x = input)[name = tensor("op_1468")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1024, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1469 = const()[name = tensor("op_1469"), val = tensor([1, 1, 12, 256])]; + tensor input = reshape(shape = var_1469, x = xt)[name = tensor("input")]; + tensor var_1471 = const()[name = tensor("op_1471"), val = tensor([-1])]; + tensor var_1472 = reduce_l2_norm(axes = var_1471, keep_dims = var_1027, x = input)[name = tensor("op_1472")]; tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; - tensor clip_13 = clip(alpha = var_1015, beta = const_42, x = var_1468)[name = tensor("clip_13")]; - tensor var_1470 = real_div(x = input, y = clip_13)[name = tensor("op_1470")]; + tensor clip_13 = clip(alpha = var_1019, beta = const_42, x = var_1472)[name = tensor("clip_13")]; + tensor var_1474 = real_div(x = input, y = clip_13)[name = tensor("op_1474")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_10 = const()[name = tensor("concat_10"), val = tensor([1, 256, 12])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1470)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_10, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_11 = const()[name = tensor("concat_11"), val = tensor([1, 256, 12])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1474)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_11, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = emb, y = reshape_1)[name = tensor("matmul_2")]; @@ -1289,10 +1298,10 @@ program(1.0) tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 1, 11])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = matmul_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_1474")]; - tensor var_1476_axis_0 = const()[name = tensor("op_1476_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_1476_axis_0, values = (var_1172, nkv))[name = tensor("op_1476")]; - tensor var_1478_axis_0 = const()[name = tensor("op_1478_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_1478_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1478")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1478")]; + tensor var_1480_axis_0 = const()[name = tensor("op_1480_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1480_axis_0, values = (var_1176, nkv))[name = tensor("op_1480")]; + tensor var_1482_axis_0 = const()[name = tensor("op_1482_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1482_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1482")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/DIHARD II/ls_eend_dih2_100ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/DIHARD II/ls_eend_dih2_100ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index b81d8b5cbe0d8098649fa47507a3021639b467ea..90bc8842ee147d0040da1a5c7c8cd536c666919a 100644 --- a/optimized/DIHARD II/ls_eend_dih2_100ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/DIHARD II/ls_eend_dih2_100ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ad90e7ac1e1c092ed0f4d317fe5e412223360188020ee69bab866499aaa197c5 -size 184735 +oid sha256:78d5f3c85bbab389dd8dd69f5a61d8f15dd3284606050d2a7740964dd402d51e +size 185993 diff --git a/optimized/DIHARD II/ls_eend_dih2_100ms.mlpackage/Manifest.json b/optimized/DIHARD II/ls_eend_dih2_100ms.mlpackage/Manifest.json index 3eef625ea9c329d45c71788044703d42d7d2d8ff..b9f285ec2e2f5ed1db07ca3762f632cdac2ea458 100644 --- a/optimized/DIHARD II/ls_eend_dih2_100ms.mlpackage/Manifest.json +++ b/optimized/DIHARD II/ls_eend_dih2_100ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "97E90057-2D2D-4D37-B3D5-77C178493AF8": { - "author": "com.apple.CoreML", - "description": "CoreML Model Weights", - "name": "weights", - "path": "com.apple.CoreML/weights" - }, - "AFFE1825-BC68-4271-9124-BAA99356C854": { + "3D1E0AB8-240F-4566-98A2-434F590DB13D": { "author": "com.apple.CoreML", "description": "CoreML Model Specification", "name": "model.mlmodel", "path": "com.apple.CoreML/model.mlmodel" + }, + "D209726A-4683-4B76-9EEA-72AD4D375EA6": { + "author": "com.apple.CoreML", + "description": "CoreML Model Weights", + "name": "weights", + "path": "com.apple.CoreML/weights" } }, - "rootModelIdentifier": "AFFE1825-BC68-4271-9124-BAA99356C854" + "rootModelIdentifier": "3D1E0AB8-240F-4566-98A2-434F590DB13D" } diff --git a/optimized/DIHARD II/ls_eend_dih2_200ms.mlmodelc/analytics/coremldata.bin b/optimized/DIHARD II/ls_eend_dih2_200ms.mlmodelc/analytics/coremldata.bin index 2fa73872bf888cb4af970615c9f3cc2833b37a00..00a4d9c8fe1abd298bb2e243e2edf587bf4dce9f 100644 --- a/optimized/DIHARD II/ls_eend_dih2_200ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/DIHARD II/ls_eend_dih2_200ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ea3e5722f9b9bb2853adb0f39a4c76699670072ba441f95ded7bfdda87bf3f9f +oid sha256:d90006dd8c55396dcd8b795c74455b263f690b78175fab492584b435b411ae5d size 243 diff --git a/optimized/DIHARD II/ls_eend_dih2_200ms.mlmodelc/coremldata.bin b/optimized/DIHARD II/ls_eend_dih2_200ms.mlmodelc/coremldata.bin index 87669cd2f920cc47225c13a1f65b100e041e3445..b376e6b64528d5838d8804f06ff5449c291fcdb0 100644 --- a/optimized/DIHARD II/ls_eend_dih2_200ms.mlmodelc/coremldata.bin +++ b/optimized/DIHARD II/ls_eend_dih2_200ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a68cbbc08dea2088e71d19677d7f8f04711af50a100a5e849e03bb6d0efa7438 -size 1266 +oid sha256:6bf619965ef450613f82b1f1a6d909445dbe9cd1dc4647577d225aa49e7cc68e +size 1357 diff --git a/optimized/DIHARD II/ls_eend_dih2_200ms.mlmodelc/metadata.json b/optimized/DIHARD II/ls_eend_dih2_200ms.mlmodelc/metadata.json index c827cb859680dca10f2d7d332df174f6c55e21e8..82b1990ae0ac64899329e4010f793efad04054e1 100644 --- a/optimized/DIHARD II/ls_eend_dih2_200ms.mlmodelc/metadata.json +++ b/optimized/DIHARD II/ls_eend_dih2_200ms.mlmodelc/metadata.json @@ -96,30 +96,30 @@ "Ios17.reshape" : 84, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 61, - "Split" : 8, + "Ios17.transpose" : 57, + "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 70, "Tile" : 9, - "Ios17.sliceByIndex" : 60, - "Ios16.sigmoid" : 9, + "Ios17.sliceByIndex" : 52, + "Ios16.sigmoid" : 5, "Ios16.reduceSum" : 1, "Ios17.squeeze" : 2, - "Ios17.layerNorm" : 37, - "Ios17.batchNorm" : 8, + "Ios17.layerNorm" : 33, + "Ios17.batchNorm" : 4, "Ios17.cast" : 16, "Ios17.sqrt" : 18, "Ios17.sub" : 16, - "Ios17.conv" : 25, + "Ios17.conv" : 13, "Ios17.realDiv" : 20, "Ios17.linear" : 56, "Ios17.concat" : 14, "Ios17.clip" : 22, "Ios17.greaterEqual" : 8, - "Ios16.silu" : 22, + "Ios16.silu" : 18, "Stack" : 6, "Ios16.cumsum" : 1, - "Ios17.mul" : 83 + "Ios17.mul" : 79 }, "computePrecision" : "Mixed (Float32, Int32)", "isUpdatable" : "0", @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"dih2\", \"model_label\": \"DIHARD II\", \"variant\": \"pipeline\", \"T\": 2, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"dih2\", \"model_label\": \"DIHARD II\", \"variant\": \"pipeline\", \"chunk_size\": 2, \"step_duration_ms\": 200, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/DIHARD II/ls_eend_dih2_200ms.mlmodelc/model.mil b/optimized/DIHARD II/ls_eend_dih2_200ms.mlmodelc/model.mil index 114ace2e9611bac24b7643b50777d9d91061c4a9..1385f671a75237f030dc475fda5fd7ef583c51e3 100644 --- a/optimized/DIHARD II/ls_eend_dih2_200ms.mlmodelc/model.mil +++ b/optimized/DIHARD II/ls_eend_dih2_200ms.mlmodelc/model.mil @@ -217,6 +217,7 @@ program(1.0) tensor var_21 = const()[name = tensor("op_21"), val = tensor(0x0p+0)]; tensor var_24 = const()[name = tensor("op_24"), val = tensor(0x1.0c6f7ap-20)]; tensor var_27 = const()[name = tensor("op_27"), val = tensor(2)]; + tensor var_30 = const()[name = tensor("op_30"), val = tensor(0)]; tensor var_33 = const()[name = tensor("op_33"), val = tensor(1)]; tensor var_35 = const()[name = tensor("op_35"), val = tensor(0x1.4f8b58p-17)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(true)]; @@ -263,8 +264,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_61")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_62")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_196 = const()[name = tensor("op_196"), val = tensor([2, 1])]; tensor var_197 = reshape(shape = var_196, x = sqrt_s_t_1)[name = tensor("op_197")]; @@ -272,7 +273,7 @@ program(1.0) tensor var_199 = mul(x = qk_1, y = M_1)[name = tensor("op_199")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_60")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_199, y = v_1)[name = tensor("inner_1")]; tensor var_201_transpose_x_0 = const()[name = tensor("op_201_transpose_x_0"), val = tensor(false)]; tensor var_201_transpose_y_0 = const()[name = tensor("op_201_transpose_y_0"), val = tensor(false)]; @@ -293,7 +294,7 @@ program(1.0) tensor var_214 = real_div(x = new_kv_unnorm_1, y = var_213)[name = tensor("op_214")]; tensor var_215_perm_0 = const()[name = tensor("op_215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_59")]; + tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_24, x = var_215)[name = tensor("out_3")]; tensor var_219 = const()[name = tensor("op_219"), val = tensor([1, 2, 256])]; tensor out_5 = reshape(shape = var_219, x = out_3)[name = tensor("out_5")]; @@ -341,145 +342,108 @@ program(1.0) tensor var_252 = sub(x = var_17, y = use_roll_1)[name = tensor("op_252")]; tensor var_253 = mul(x = filled_window_1, y = var_252)[name = tensor("op_253")]; tensor var_254 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_254")]; - tensor input_21 = add(x = var_253, y = var_254)[name = tensor("input_21")]; + tensor window_3 = add(x = var_253, y = var_254)[name = tensor("window_3")]; tensor var_256 = const()[name = tensor("op_256"), val = tensor(0x1p+0)]; tensor var_257 = add(x = filled_1, y = var_256)[name = tensor("op_257")]; tensor clip_1 = clip(alpha = var_21, beta = var_16, x = var_257)[name = tensor("clip_1")]; + tensor var_260_begin_0 = const()[name = tensor("op_260_begin_0"), val = tensor([0, 1, 0])]; + tensor var_260_end_0 = const()[name = tensor("op_260_end_0"), val = tensor([1, 1, 256])]; + tensor var_260_end_mask_0 = const()[name = tensor("op_260_end_mask_0"), val = tensor([true, true, true])]; + tensor var_260 = slice_by_index(begin = var_260_begin_0, end = var_260_end_0, end_mask = var_260_end_mask_0, x = x_3)[name = tensor("op_260")]; + tensor var_263_begin_0 = const()[name = tensor("op_263_begin_0"), val = tensor([0, 1, 0])]; + tensor var_263_end_0 = const()[name = tensor("op_263_end_0"), val = tensor([1, 16, 256])]; + tensor var_263_end_mask_0 = const()[name = tensor("op_263_end_mask_0"), val = tensor([true, true, true])]; + tensor var_263 = slice_by_index(begin = var_263_begin_0, end = var_263_end_0, end_mask = var_263_end_mask_0, x = window_3)[name = tensor("op_263")]; + tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; + tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_263, var_260))[name = tensor("rolled_3")]; + tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; + tensor var_268 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_268")]; + tensor var_269_dtype_0 = const()[name = tensor("op_269_dtype_0"), val = tensor("fp32")]; + tensor var_270 = const()[name = tensor("op_270"), val = tensor([1, 16, 1])]; + tensor var_269 = cast(dtype = var_269_dtype_0, x = var_268)[name = tensor("cast_141")]; + tensor fill_mask_3 = reshape(shape = var_270, x = var_269)[name = tensor("fill_mask_3")]; + tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_260)[name = tensor("x_expanded_3")]; + tensor var_274 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_274")]; + tensor var_275 = mul(x = window_3, y = var_274)[name = tensor("op_275")]; + tensor var_276 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_276")]; + tensor filled_window_3 = add(x = var_275, y = var_276)[name = tensor("filled_window_3")]; + tensor var_278 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_278")]; + tensor var_279_dtype_0 = const()[name = tensor("op_279_dtype_0"), val = tensor("fp32")]; + tensor var_280 = const()[name = tensor("op_280"), val = tensor([1, 1, 1])]; + tensor var_279 = cast(dtype = var_279_dtype_0, x = var_278)[name = tensor("cast_140")]; + tensor use_roll_3 = reshape(shape = var_280, x = var_279)[name = tensor("use_roll_3")]; + tensor var_282 = sub(x = var_17, y = use_roll_3)[name = tensor("op_282")]; + tensor var_283 = mul(x = filled_window_3, y = var_282)[name = tensor("op_283")]; + tensor var_284 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_284")]; + tensor window_5 = add(x = var_283, y = var_284)[name = tensor("window_5")]; + tensor var_286 = const()[name = tensor("op_286"), val = tensor(0x1p+0)]; + tensor var_287 = add(x = clip_1, y = var_286)[name = tensor("op_287")]; + tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_287)[name = tensor("clip_3")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_30, interleave = input_21_interleave_0, values = (window_3, window_5))[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; - tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; + tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; tensor inputs_3_pad_type_0 = const()[name = tensor("inputs_3_pad_type_0"), val = tensor("valid")]; tensor inputs_3_strides_0 = const()[name = tensor("inputs_3_strides_0"), val = tensor([1])]; tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_58")]; - tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_278_split_sizes_0 = const()[name = tensor("op_278_split_sizes_0"), val = tensor([256, 256])]; - tensor var_278_axis_0 = const()[name = tensor("op_278_axis_0"), val = tensor(1)]; - tensor var_278_0, tensor var_278_1 = split(axis = var_278_axis_0, split_sizes = var_278_split_sizes_0, x = inputs_3)[name = tensor("op_278")]; - tensor var_280 = sigmoid(x = var_278_1)[name = tensor("op_280")]; - tensor inputs_5 = mul(x = var_278_0, y = var_280)[name = tensor("inputs_5")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; + tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; + tensor var_310_split_sizes_0 = const()[name = tensor("op_310_split_sizes_0"), val = tensor([256, 256])]; + tensor var_310_axis_0 = const()[name = tensor("op_310_axis_0"), val = tensor(1)]; + tensor var_310_0, tensor var_310_1 = split(axis = var_310_axis_0, split_sizes = var_310_split_sizes_0, x = inputs_3)[name = tensor("op_310")]; + tensor var_312 = sigmoid(x = var_310_1)[name = tensor("op_312")]; + tensor inputs_5 = mul(x = var_310_0, y = var_312)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; tensor outputs_aug_1_strides_0 = const()[name = tensor("outputs_aug_1_strides_0"), val = tensor([1])]; tensor outputs_aug_1_dilations_0 = const()[name = tensor("outputs_aug_1_dilations_0"), val = tensor([1])]; - tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; + tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; tensor input_25_begin_0 = const()[name = tensor("input_25_begin_0"), val = tensor([0, 0, 0])]; - tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([1, 256, 16])]; + tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([2, 256, 16])]; tensor input_25_end_mask_0 = const()[name = tensor("input_25_end_mask_0"), val = tensor([true, true, false])]; - tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; - tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; - tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; + tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; + tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; + tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; tensor input_29_pad_type_0 = const()[name = tensor("input_29_pad_type_0"), val = tensor("valid")]; tensor input_29_strides_0 = const()[name = tensor("input_29_strides_0"), val = tensor([1])]; tensor input_29_pad_0 = const()[name = tensor("input_29_pad_0"), val = tensor([0, 0])]; tensor input_29_dilations_0 = const()[name = tensor("input_29_dilations_0"), val = tensor([1])]; tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; - tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; + tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_311_begin_0 = const()[name = tensor("op_311_begin_0"), val = tensor([0, -1, 0])]; - tensor var_311_end_0 = const()[name = tensor("op_311_end_0"), val = tensor([1, 16, 256])]; - tensor var_311_end_mask_0 = const()[name = tensor("op_311_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_57")]; - tensor var_311 = slice_by_index(begin = var_311_begin_0, end = var_311_end_0, end_mask = var_311_end_mask_0, x = conv_out_1)[name = tensor("op_311")]; - tensor var_314_begin_0 = const()[name = tensor("op_314_begin_0"), val = tensor([0, 1, 0])]; - tensor var_314_end_0 = const()[name = tensor("op_314_end_0"), val = tensor([1, 1, 256])]; - tensor var_314_end_mask_0 = const()[name = tensor("op_314_end_mask_0"), val = tensor([true, true, true])]; - tensor var_314 = slice_by_index(begin = var_314_begin_0, end = var_314_end_0, end_mask = var_314_end_mask_0, x = x_3)[name = tensor("op_314")]; - tensor var_317_begin_0 = const()[name = tensor("op_317_begin_0"), val = tensor([0, 1, 0])]; - tensor var_317_end_0 = const()[name = tensor("op_317_end_0"), val = tensor([1, 16, 256])]; - tensor var_317_end_mask_0 = const()[name = tensor("op_317_end_mask_0"), val = tensor([true, true, true])]; - tensor var_317 = slice_by_index(begin = var_317_begin_0, end = var_317_end_0, end_mask = var_317_end_mask_0, x = input_21)[name = tensor("op_317")]; - tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_317, var_314))[name = tensor("rolled_3")]; - tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; - tensor var_322 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_322")]; - tensor var_323_dtype_0 = const()[name = tensor("op_323_dtype_0"), val = tensor("fp32")]; - tensor var_324 = const()[name = tensor("op_324"), val = tensor([1, 16, 1])]; - tensor var_323 = cast(dtype = var_323_dtype_0, x = var_322)[name = tensor("cast_141")]; - tensor fill_mask_3 = reshape(shape = var_324, x = var_323)[name = tensor("fill_mask_3")]; - tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_314)[name = tensor("x_expanded_3")]; - tensor var_328 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_328")]; - tensor var_329 = mul(x = input_21, y = var_328)[name = tensor("op_329")]; - tensor var_330 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_330")]; - tensor filled_window_3 = add(x = var_329, y = var_330)[name = tensor("filled_window_3")]; - tensor var_332 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_332")]; - tensor var_333_dtype_0 = const()[name = tensor("op_333_dtype_0"), val = tensor("fp32")]; - tensor var_334 = const()[name = tensor("op_334"), val = tensor([1, 1, 1])]; - tensor var_333 = cast(dtype = var_333_dtype_0, x = var_332)[name = tensor("cast_140")]; - tensor use_roll_3 = reshape(shape = var_334, x = var_333)[name = tensor("use_roll_3")]; - tensor var_336 = sub(x = var_17, y = use_roll_3)[name = tensor("op_336")]; - tensor var_337 = mul(x = filled_window_3, y = var_336)[name = tensor("op_337")]; - tensor var_338 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_338")]; - tensor input_31 = add(x = var_337, y = var_338)[name = tensor("input_31")]; - tensor var_340 = const()[name = tensor("op_340"), val = tensor(0x1p+0)]; - tensor var_341 = add(x = clip_1, y = var_340)[name = tensor("op_341")]; - tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_341)[name = tensor("clip_3")]; - tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; - tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_31)[name = tensor("x_7")]; - tensor input_33_perm_0 = const()[name = tensor("input_33_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_9_pad_type_0 = const()[name = tensor("inputs_9_pad_type_0"), val = tensor("valid")]; - tensor inputs_9_strides_0 = const()[name = tensor("inputs_9_strides_0"), val = tensor([1])]; - tensor inputs_9_pad_0 = const()[name = tensor("inputs_9_pad_0"), val = tensor([0, 0])]; - tensor inputs_9_dilations_0 = const()[name = tensor("inputs_9_dilations_0"), val = tensor([1])]; - tensor inputs_9_groups_0 = const()[name = tensor("inputs_9_groups_0"), val = tensor(1)]; - tensor input_33 = transpose(perm = input_33_perm_0, x = x_7)[name = tensor("transpose_56")]; - tensor inputs_9 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_9_dilations_0, groups = inputs_9_groups_0, pad = inputs_9_pad_0, pad_type = inputs_9_pad_type_0, strides = inputs_9_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_33)[name = tensor("inputs_9")]; - tensor var_362_split_sizes_0 = const()[name = tensor("op_362_split_sizes_0"), val = tensor([256, 256])]; - tensor var_362_axis_0 = const()[name = tensor("op_362_axis_0"), val = tensor(1)]; - tensor var_362_0, tensor var_362_1 = split(axis = var_362_axis_0, split_sizes = var_362_split_sizes_0, x = inputs_9)[name = tensor("op_362")]; - tensor var_364 = sigmoid(x = var_362_1)[name = tensor("op_364")]; - tensor inputs_11 = mul(x = var_362_0, y = var_364)[name = tensor("inputs_11")]; - tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; - tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; - tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; - tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_11)[name = tensor("outputs_aug_3")]; - tensor input_35_begin_0 = const()[name = tensor("input_35_begin_0"), val = tensor([0, 0, 0])]; - tensor input_35_end_0 = const()[name = tensor("input_35_end_0"), val = tensor([1, 256, 16])]; - tensor input_35_end_mask_0 = const()[name = tensor("input_35_end_mask_0"), val = tensor([true, true, false])]; - tensor input_35 = slice_by_index(begin = input_35_begin_0, end = input_35_end_0, end_mask = input_35_end_mask_0, x = outputs_aug_3)[name = tensor("input_35")]; - tensor inputs_13 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_35)[name = tensor("inputs_13")]; - tensor input_37 = silu(x = inputs_13)[name = tensor("input_37")]; - tensor input_39_pad_type_0 = const()[name = tensor("input_39_pad_type_0"), val = tensor("valid")]; - tensor input_39_strides_0 = const()[name = tensor("input_39_strides_0"), val = tensor([1])]; - tensor input_39_pad_0 = const()[name = tensor("input_39_pad_0"), val = tensor([0, 0])]; - tensor input_39_dilations_0 = const()[name = tensor("input_39_dilations_0"), val = tensor([1])]; - tensor input_39_groups_0 = const()[name = tensor("input_39_groups_0"), val = tensor(1)]; - tensor input_39 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_37)[name = tensor("input_39")]; - tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_395_begin_0 = const()[name = tensor("op_395_begin_0"), val = tensor([0, -1, 0])]; - tensor var_395_end_0 = const()[name = tensor("op_395_end_0"), val = tensor([1, 16, 256])]; - tensor var_395_end_mask_0 = const()[name = tensor("op_395_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_39)[name = tensor("transpose_55")]; - tensor var_395 = slice_by_index(begin = var_395_begin_0, end = var_395_end_0, end_mask = var_395_end_mask_0, x = conv_out_3)[name = tensor("op_395")]; - tensor var_398_interleave_0 = const()[name = tensor("op_398_interleave_0"), val = tensor(false)]; - tensor var_398 = concat(axis = var_33, interleave = var_398_interleave_0, values = (var_311, var_395))[name = tensor("op_398")]; - tensor input_41 = add(x = x_3, y = var_398)[name = tensor("input_41")]; + tensor var_343_begin_0 = const()[name = tensor("op_343_begin_0"), val = tensor([0, -1, 0])]; + tensor var_343_end_0 = const()[name = tensor("op_343_end_0"), val = tensor([2, 16, 256])]; + tensor var_343_end_mask_0 = const()[name = tensor("op_343_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_343 = slice_by_index(begin = var_343_begin_0, end = var_343_end_0, end_mask = var_343_end_mask_0, x = conv_out_1)[name = tensor("op_343")]; + tensor var_345_perm_0 = const()[name = tensor("op_345_perm_0"), val = tensor([1, 0, 2])]; + tensor var_345 = transpose(perm = var_345_perm_0, x = var_343)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_345)[name = tensor("input_31")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; + tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; + tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; + tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; + tensor var_368 = const()[name = tensor("op_368"), val = tensor(0x1p-1)]; + tensor var_369 = mul(x = input_39, y = var_368)[name = tensor("op_369")]; + tensor input_41 = add(x = var_369, y = input_31)[name = tensor("input_41")]; tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; - tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_41)[name = tensor("input_43")]; - tensor inputs_15 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_43)[name = tensor("linear_8")]; - tensor input_45 = silu(x = inputs_15)[name = tensor("input_45")]; - tensor input_49 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_45)[name = tensor("linear_9")]; - tensor var_421 = const()[name = tensor("op_421"), val = tensor(0x1p-1)]; - tensor var_422 = mul(x = input_49, y = var_421)[name = tensor("op_422")]; - tensor input_51 = add(x = var_422, y = input_41)[name = tensor("input_51")]; - tensor input_53_axes_0 = const()[name = tensor("input_53_axes_0"), val = tensor([-1])]; - tensor input_53 = layer_norm(axes = input_53_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_51)[name = tensor("input_53")]; - tensor input_55_axes_0 = const()[name = tensor("input_55_axes_0"), val = tensor([-1])]; - tensor input_55 = layer_norm(axes = input_55_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_53)[name = tensor("input_55")]; - tensor inputs_17 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_55)[name = tensor("linear_10")]; - tensor input_57 = silu(x = inputs_17)[name = tensor("input_57")]; - tensor input_61 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_57)[name = tensor("linear_11")]; - tensor var_451 = const()[name = tensor("op_451"), val = tensor(0x1p-1)]; - tensor var_452 = mul(x = input_61, y = var_451)[name = tensor("op_452")]; - tensor input_63 = add(x = var_452, y = input_53)[name = tensor("input_63")]; - tensor x_9_axes_0 = const()[name = tensor("x_9_axes_0"), val = tensor([-1])]; - tensor x_9 = layer_norm(axes = x_9_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_63)[name = tensor("x_9")]; + tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; + tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; + tensor input_45 = layer_norm(axes = input_45_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_43)[name = tensor("input_45")]; + tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; + tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; + tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; + tensor var_398 = const()[name = tensor("op_398"), val = tensor(0x1p-1)]; + tensor var_399 = mul(x = input_51, y = var_398)[name = tensor("op_399")]; + tensor input_53 = add(x = var_399, y = input_43)[name = tensor("input_53")]; + tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; + tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_3_end_0 = const()[name = tensor("prev_kv_3_end_0"), val = tensor([2, 1, 4, 64, 64])]; tensor prev_kv_3_end_mask_0 = const()[name = tensor("prev_kv_3_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -490,243 +454,206 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_466 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_9)[name = tensor("linear_12")]; - tensor var_467 = const()[name = tensor("op_467"), val = tensor([1, 2, 4, 64])]; - tensor var_468 = reshape(shape = var_467, x = var_466)[name = tensor("op_468")]; + tensor var_413 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_414 = const()[name = tensor("op_414"), val = tensor([1, 2, 4, 64])]; + tensor var_415 = reshape(shape = var_414, x = var_413)[name = tensor("op_415")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_472 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_9)[name = tensor("linear_13")]; - tensor var_473 = const()[name = tensor("op_473"), val = tensor(0x1p-3)]; - tensor var_474 = mul(x = var_472, y = var_473)[name = tensor("op_474")]; - tensor var_475 = const()[name = tensor("op_475"), val = tensor([1, 2, 4, 64])]; - tensor var_476 = reshape(shape = var_475, x = var_474)[name = tensor("op_476")]; + tensor var_419 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_420 = const()[name = tensor("op_420"), val = tensor(0x1p-3)]; + tensor var_421 = mul(x = var_419, y = var_420)[name = tensor("op_421")]; + tensor var_422 = const()[name = tensor("op_422"), val = tensor([1, 2, 4, 64])]; + tensor var_423 = reshape(shape = var_422, x = var_421)[name = tensor("op_423")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_480 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_9)[name = tensor("linear_14")]; - tensor var_481 = const()[name = tensor("op_481"), val = tensor([1, 2, 4, 64])]; - tensor var_482 = reshape(shape = var_481, x = var_480)[name = tensor("op_482")]; + tensor var_427 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_428 = const()[name = tensor("op_428"), val = tensor([1, 2, 4, 64])]; + tensor var_429 = reshape(shape = var_428, x = var_427)[name = tensor("op_429")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_67 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_9)[name = tensor("linear_15")]; + tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; tensor s_t_3 = add(x = prev_scale_3, y = encoder__t_index)[name = tensor("s_t_3")]; tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_476)[name = tensor("transpose_53")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_468)[name = tensor("transpose_54")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_423)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_415)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_492 = const()[name = tensor("op_492"), val = tensor([2, 1])]; - tensor var_493 = reshape(shape = var_492, x = sqrt_s_t_3)[name = tensor("op_493")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_493)[name = tensor("M_3")]; - tensor var_495 = mul(x = qk_3, y = M_3)[name = tensor("op_495")]; + tensor var_439 = const()[name = tensor("op_439"), val = tensor([2, 1])]; + tensor var_440 = reshape(shape = var_439, x = sqrt_s_t_3)[name = tensor("op_440")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_440)[name = tensor("M_3")]; + tensor var_442 = mul(x = qk_3, y = M_3)[name = tensor("op_442")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_482)[name = tensor("transpose_52")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_495, y = v_3)[name = tensor("inner_3")]; - tensor var_497_transpose_x_0 = const()[name = tensor("op_497_transpose_x_0"), val = tensor(false)]; - tensor var_497_transpose_y_0 = const()[name = tensor("op_497_transpose_y_0"), val = tensor(false)]; - tensor var_497 = matmul(transpose_x = var_497_transpose_x_0, transpose_y = var_497_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_497")]; - tensor var_498 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_498")]; - tensor var_499 = const()[name = tensor("op_499"), val = tensor([1, 1, 2, 1])]; - tensor var_500 = reshape(shape = var_499, x = var_498)[name = tensor("op_500")]; - tensor cross_3 = mul(x = var_497, y = var_500)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_429)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_442, y = v_3)[name = tensor("inner_3")]; + tensor var_444_transpose_x_0 = const()[name = tensor("op_444_transpose_x_0"), val = tensor(false)]; + tensor var_444_transpose_y_0 = const()[name = tensor("op_444_transpose_y_0"), val = tensor(false)]; + tensor var_444 = matmul(transpose_x = var_444_transpose_x_0, transpose_y = var_444_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_444")]; + tensor var_445 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_445")]; + tensor var_446 = const()[name = tensor("op_446"), val = tensor([1, 1, 2, 1])]; + tensor var_447 = reshape(shape = var_446, x = var_445)[name = tensor("op_447")]; + tensor cross_3 = mul(x = var_444, y = var_447)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_503 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_503")]; - tensor var_505_transpose_x_1 = const()[name = tensor("op_505_transpose_x_1"), val = tensor(true)]; - tensor var_505_transpose_y_1 = const()[name = tensor("op_505_transpose_y_1"), val = tensor(false)]; - tensor var_505 = matmul(transpose_x = var_505_transpose_x_1, transpose_y = var_505_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_505")]; - tensor new_kv_unnorm_3 = add(x = var_503, y = var_505)[name = tensor("new_kv_unnorm_3")]; - tensor var_507 = const()[name = tensor("op_507"), val = tensor(0x1p+1)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_507)[name = tensor("new_scale_3")]; - tensor var_509 = sqrt(x = new_scale_3)[name = tensor("op_509")]; - tensor var_510 = real_div(x = new_kv_unnorm_3, y = var_509)[name = tensor("op_510")]; - tensor var_511_perm_0 = const()[name = tensor("op_511_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_450 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_450")]; + tensor var_452_transpose_x_1 = const()[name = tensor("op_452_transpose_x_1"), val = tensor(true)]; + tensor var_452_transpose_y_1 = const()[name = tensor("op_452_transpose_y_1"), val = tensor(false)]; + tensor var_452 = matmul(transpose_x = var_452_transpose_x_1, transpose_y = var_452_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_452")]; + tensor new_kv_unnorm_3 = add(x = var_450, y = var_452)[name = tensor("new_kv_unnorm_3")]; + tensor var_454 = const()[name = tensor("op_454"), val = tensor(0x1p+1)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_454)[name = tensor("new_scale_3")]; + tensor var_456 = sqrt(x = new_scale_3)[name = tensor("op_456")]; + tensor var_457 = real_div(x = new_kv_unnorm_3, y = var_456)[name = tensor("op_457")]; + tensor var_458_perm_0 = const()[name = tensor("op_458_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_511 = transpose(perm = var_511_perm_0, x = out_7)[name = tensor("transpose_51")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_511)[name = tensor("out_9")]; - tensor var_515 = const()[name = tensor("op_515"), val = tensor([1, 2, 256])]; - tensor out_11 = reshape(shape = var_515, x = out_9)[name = tensor("out_11")]; - tensor var_517 = silu(x = input_67)[name = tensor("op_517")]; - tensor input_69 = mul(x = var_517, y = out_11)[name = tensor("input_69")]; - tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_69)[name = tensor("linear_16")]; - tensor x_11 = add(x = input_63, y = ret_out_3)[name = tensor("x_11")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor var_458 = transpose(perm = var_458_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_458)[name = tensor("out_9")]; + tensor var_462 = const()[name = tensor("op_462"), val = tensor([1, 2, 256])]; + tensor out_11 = reshape(shape = var_462, x = out_9)[name = tensor("out_11")]; + tensor var_464 = silu(x = input_57)[name = tensor("op_464")]; + tensor input_59 = mul(x = var_464, y = out_11)[name = tensor("input_59")]; + tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; + tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; + tensor window_7_begin_0 = const()[name = tensor("window_7_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_7_end_0 = const()[name = tensor("window_7_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_7_end_mask_0 = const()[name = tensor("window_7_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_7_squeeze_mask_0 = const()[name = tensor("window_7_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_7 = slice_by_index(begin = window_7_begin_0, end = window_7_end_0, end_mask = window_7_end_mask_0, squeeze_mask = window_7_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_7")]; tensor filled_5_begin_0 = const()[name = tensor("filled_5_begin_0"), val = tensor([1, 0])]; tensor filled_5_end_0 = const()[name = tensor("filled_5_end_0"), val = tensor([2, 1])]; tensor filled_5_end_mask_0 = const()[name = tensor("filled_5_end_mask_0"), val = tensor([false, true])]; tensor filled_5_squeeze_mask_0 = const()[name = tensor("filled_5_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_5 = slice_by_index(begin = filled_5_begin_0, end = filled_5_end_0, end_mask = filled_5_end_mask_0, squeeze_mask = filled_5_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_5")]; - tensor var_526_begin_0 = const()[name = tensor("op_526_begin_0"), val = tensor([0, 0, 0])]; - tensor var_526_end_0 = const()[name = tensor("op_526_end_0"), val = tensor([1, 1, 256])]; - tensor var_526_end_mask_0 = const()[name = tensor("op_526_end_mask_0"), val = tensor([true, false, true])]; - tensor var_526 = slice_by_index(begin = var_526_begin_0, end = var_526_end_0, end_mask = var_526_end_mask_0, x = x_11)[name = tensor("op_526")]; - tensor var_529_begin_0 = const()[name = tensor("op_529_begin_0"), val = tensor([0, 1, 0])]; - tensor var_529_end_0 = const()[name = tensor("op_529_end_0"), val = tensor([1, 16, 256])]; - tensor var_529_end_mask_0 = const()[name = tensor("op_529_end_mask_0"), val = tensor([true, true, true])]; - tensor var_529 = slice_by_index(begin = var_529_begin_0, end = var_529_end_0, end_mask = var_529_end_mask_0, x = window_3)[name = tensor("op_529")]; + tensor var_473_begin_0 = const()[name = tensor("op_473_begin_0"), val = tensor([0, 0, 0])]; + tensor var_473_end_0 = const()[name = tensor("op_473_end_0"), val = tensor([1, 1, 256])]; + tensor var_473_end_mask_0 = const()[name = tensor("op_473_end_mask_0"), val = tensor([true, false, true])]; + tensor var_473 = slice_by_index(begin = var_473_begin_0, end = var_473_end_0, end_mask = var_473_end_mask_0, x = x_9)[name = tensor("op_473")]; + tensor var_476_begin_0 = const()[name = tensor("op_476_begin_0"), val = tensor([0, 1, 0])]; + tensor var_476_end_0 = const()[name = tensor("op_476_end_0"), val = tensor([1, 16, 256])]; + tensor var_476_end_mask_0 = const()[name = tensor("op_476_end_mask_0"), val = tensor([true, true, true])]; + tensor var_476 = slice_by_index(begin = var_476_begin_0, end = var_476_end_0, end_mask = var_476_end_mask_0, x = window_7)[name = tensor("op_476")]; tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_529, var_526))[name = tensor("rolled_5")]; + tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_476, var_473))[name = tensor("rolled_5")]; tensor clip_4 = clip(alpha = var_21, beta = var_20, x = filled_5)[name = tensor("clip_4")]; - tensor var_534 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_534")]; - tensor var_535_dtype_0 = const()[name = tensor("op_535_dtype_0"), val = tensor("fp32")]; - tensor var_536 = const()[name = tensor("op_536"), val = tensor([1, 16, 1])]; - tensor var_535 = cast(dtype = var_535_dtype_0, x = var_534)[name = tensor("cast_139")]; - tensor fill_mask_5 = reshape(shape = var_536, x = var_535)[name = tensor("fill_mask_5")]; + tensor var_481 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_481")]; + tensor var_482_dtype_0 = const()[name = tensor("op_482_dtype_0"), val = tensor("fp32")]; + tensor var_483 = const()[name = tensor("op_483"), val = tensor([1, 16, 1])]; + tensor var_482 = cast(dtype = var_482_dtype_0, x = var_481)[name = tensor("cast_139")]; + tensor fill_mask_5 = reshape(shape = var_483, x = var_482)[name = tensor("fill_mask_5")]; tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_526)[name = tensor("x_expanded_5")]; - tensor var_540 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_540")]; - tensor var_541 = mul(x = window_3, y = var_540)[name = tensor("op_541")]; - tensor var_542 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_542")]; - tensor filled_window_5 = add(x = var_541, y = var_542)[name = tensor("filled_window_5")]; - tensor var_544 = greater_equal(x = filled_5, y = var_16)[name = tensor("op_544")]; - tensor var_545_dtype_0 = const()[name = tensor("op_545_dtype_0"), val = tensor("fp32")]; - tensor var_546 = const()[name = tensor("op_546"), val = tensor([1, 1, 1])]; - tensor var_545 = cast(dtype = var_545_dtype_0, x = var_544)[name = tensor("cast_138")]; - tensor use_roll_5 = reshape(shape = var_546, x = var_545)[name = tensor("use_roll_5")]; - tensor var_548 = sub(x = var_17, y = use_roll_5)[name = tensor("op_548")]; - tensor var_549 = mul(x = filled_window_5, y = var_548)[name = tensor("op_549")]; - tensor var_550 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_550")]; - tensor input_71 = add(x = var_549, y = var_550)[name = tensor("input_71")]; - tensor var_552 = const()[name = tensor("op_552"), val = tensor(0x1p+0)]; - tensor var_553 = add(x = filled_5, y = var_552)[name = tensor("op_553")]; - tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_553)[name = tensor("clip_5")]; - tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; - tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_71)[name = tensor("x_13")]; - tensor input_73_perm_0 = const()[name = tensor("input_73_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_19_pad_type_0 = const()[name = tensor("inputs_19_pad_type_0"), val = tensor("valid")]; - tensor inputs_19_strides_0 = const()[name = tensor("inputs_19_strides_0"), val = tensor([1])]; - tensor inputs_19_pad_0 = const()[name = tensor("inputs_19_pad_0"), val = tensor([0, 0])]; - tensor inputs_19_dilations_0 = const()[name = tensor("inputs_19_dilations_0"), val = tensor([1])]; - tensor inputs_19_groups_0 = const()[name = tensor("inputs_19_groups_0"), val = tensor(1)]; - tensor input_73 = transpose(perm = input_73_perm_0, x = x_13)[name = tensor("transpose_50")]; - tensor inputs_19 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_19_dilations_0, groups = inputs_19_groups_0, pad = inputs_19_pad_0, pad_type = inputs_19_pad_type_0, strides = inputs_19_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_73)[name = tensor("inputs_19")]; - tensor var_574_split_sizes_0 = const()[name = tensor("op_574_split_sizes_0"), val = tensor([256, 256])]; - tensor var_574_axis_0 = const()[name = tensor("op_574_axis_0"), val = tensor(1)]; - tensor var_574_0, tensor var_574_1 = split(axis = var_574_axis_0, split_sizes = var_574_split_sizes_0, x = inputs_19)[name = tensor("op_574")]; - tensor var_576 = sigmoid(x = var_574_1)[name = tensor("op_576")]; - tensor inputs_21 = mul(x = var_574_0, y = var_576)[name = tensor("inputs_21")]; - tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; - tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; - tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; - tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_21)[name = tensor("outputs_aug_5")]; - tensor input_75_begin_0 = const()[name = tensor("input_75_begin_0"), val = tensor([0, 0, 0])]; - tensor input_75_end_0 = const()[name = tensor("input_75_end_0"), val = tensor([1, 256, 16])]; - tensor input_75_end_mask_0 = const()[name = tensor("input_75_end_mask_0"), val = tensor([true, true, false])]; - tensor input_75 = slice_by_index(begin = input_75_begin_0, end = input_75_end_0, end_mask = input_75_end_mask_0, x = outputs_aug_5)[name = tensor("input_75")]; - tensor inputs_23 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_75)[name = tensor("inputs_23")]; - tensor input_77 = silu(x = inputs_23)[name = tensor("input_77")]; - tensor input_79_pad_type_0 = const()[name = tensor("input_79_pad_type_0"), val = tensor("valid")]; - tensor input_79_strides_0 = const()[name = tensor("input_79_strides_0"), val = tensor([1])]; - tensor input_79_pad_0 = const()[name = tensor("input_79_pad_0"), val = tensor([0, 0])]; - tensor input_79_dilations_0 = const()[name = tensor("input_79_dilations_0"), val = tensor([1])]; - tensor input_79_groups_0 = const()[name = tensor("input_79_groups_0"), val = tensor(1)]; - tensor input_79 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_79_dilations_0, groups = input_79_groups_0, pad = input_79_pad_0, pad_type = input_79_pad_type_0, strides = input_79_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_77)[name = tensor("input_79")]; - tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_607_begin_0 = const()[name = tensor("op_607_begin_0"), val = tensor([0, -1, 0])]; - tensor var_607_end_0 = const()[name = tensor("op_607_end_0"), val = tensor([1, 16, 256])]; - tensor var_607_end_mask_0 = const()[name = tensor("op_607_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_79)[name = tensor("transpose_49")]; - tensor var_607 = slice_by_index(begin = var_607_begin_0, end = var_607_end_0, end_mask = var_607_end_mask_0, x = conv_out_5)[name = tensor("op_607")]; - tensor var_610_begin_0 = const()[name = tensor("op_610_begin_0"), val = tensor([0, 1, 0])]; - tensor var_610_end_0 = const()[name = tensor("op_610_end_0"), val = tensor([1, 1, 256])]; - tensor var_610_end_mask_0 = const()[name = tensor("op_610_end_mask_0"), val = tensor([true, true, true])]; - tensor var_610 = slice_by_index(begin = var_610_begin_0, end = var_610_end_0, end_mask = var_610_end_mask_0, x = x_11)[name = tensor("op_610")]; - tensor var_613_begin_0 = const()[name = tensor("op_613_begin_0"), val = tensor([0, 1, 0])]; - tensor var_613_end_0 = const()[name = tensor("op_613_end_0"), val = tensor([1, 16, 256])]; - tensor var_613_end_mask_0 = const()[name = tensor("op_613_end_mask_0"), val = tensor([true, true, true])]; - tensor var_613 = slice_by_index(begin = var_613_begin_0, end = var_613_end_0, end_mask = var_613_end_mask_0, x = input_71)[name = tensor("op_613")]; + tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_473)[name = tensor("x_expanded_5")]; + tensor var_487 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_487")]; + tensor var_488 = mul(x = window_7, y = var_487)[name = tensor("op_488")]; + tensor var_489 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_489")]; + tensor filled_window_5 = add(x = var_488, y = var_489)[name = tensor("filled_window_5")]; + tensor var_491 = greater_equal(x = filled_5, y = var_16)[name = tensor("op_491")]; + tensor var_492_dtype_0 = const()[name = tensor("op_492_dtype_0"), val = tensor("fp32")]; + tensor var_493 = const()[name = tensor("op_493"), val = tensor([1, 1, 1])]; + tensor var_492 = cast(dtype = var_492_dtype_0, x = var_491)[name = tensor("cast_138")]; + tensor use_roll_5 = reshape(shape = var_493, x = var_492)[name = tensor("use_roll_5")]; + tensor var_495 = sub(x = var_17, y = use_roll_5)[name = tensor("op_495")]; + tensor var_496 = mul(x = filled_window_5, y = var_495)[name = tensor("op_496")]; + tensor var_497 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_497")]; + tensor window_9 = add(x = var_496, y = var_497)[name = tensor("window_9")]; + tensor var_499 = const()[name = tensor("op_499"), val = tensor(0x1p+0)]; + tensor var_500 = add(x = filled_5, y = var_499)[name = tensor("op_500")]; + tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_500)[name = tensor("clip_5")]; + tensor var_503_begin_0 = const()[name = tensor("op_503_begin_0"), val = tensor([0, 1, 0])]; + tensor var_503_end_0 = const()[name = tensor("op_503_end_0"), val = tensor([1, 1, 256])]; + tensor var_503_end_mask_0 = const()[name = tensor("op_503_end_mask_0"), val = tensor([true, true, true])]; + tensor var_503 = slice_by_index(begin = var_503_begin_0, end = var_503_end_0, end_mask = var_503_end_mask_0, x = x_9)[name = tensor("op_503")]; + tensor var_506_begin_0 = const()[name = tensor("op_506_begin_0"), val = tensor([0, 1, 0])]; + tensor var_506_end_0 = const()[name = tensor("op_506_end_0"), val = tensor([1, 16, 256])]; + tensor var_506_end_mask_0 = const()[name = tensor("op_506_end_mask_0"), val = tensor([true, true, true])]; + tensor var_506 = slice_by_index(begin = var_506_begin_0, end = var_506_end_0, end_mask = var_506_end_mask_0, x = window_9)[name = tensor("op_506")]; tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; - tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_613, var_610))[name = tensor("rolled_7")]; + tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_506, var_503))[name = tensor("rolled_7")]; tensor clip_6 = clip(alpha = var_21, beta = var_20, x = clip_5)[name = tensor("clip_6")]; - tensor var_618 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_618")]; - tensor var_619_dtype_0 = const()[name = tensor("op_619_dtype_0"), val = tensor("fp32")]; - tensor var_620 = const()[name = tensor("op_620"), val = tensor([1, 16, 1])]; - tensor var_619 = cast(dtype = var_619_dtype_0, x = var_618)[name = tensor("cast_137")]; - tensor fill_mask_7 = reshape(shape = var_620, x = var_619)[name = tensor("fill_mask_7")]; + tensor var_511 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_511")]; + tensor var_512_dtype_0 = const()[name = tensor("op_512_dtype_0"), val = tensor("fp32")]; + tensor var_513 = const()[name = tensor("op_513"), val = tensor([1, 16, 1])]; + tensor var_512 = cast(dtype = var_512_dtype_0, x = var_511)[name = tensor("cast_137")]; + tensor fill_mask_7 = reshape(shape = var_513, x = var_512)[name = tensor("fill_mask_7")]; tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_610)[name = tensor("x_expanded_7")]; - tensor var_624 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_624")]; - tensor var_625 = mul(x = input_71, y = var_624)[name = tensor("op_625")]; - tensor var_626 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_626")]; - tensor filled_window_7 = add(x = var_625, y = var_626)[name = tensor("filled_window_7")]; - tensor var_628 = greater_equal(x = clip_5, y = var_16)[name = tensor("op_628")]; - tensor var_629_dtype_0 = const()[name = tensor("op_629_dtype_0"), val = tensor("fp32")]; - tensor var_630 = const()[name = tensor("op_630"), val = tensor([1, 1, 1])]; - tensor var_629 = cast(dtype = var_629_dtype_0, x = var_628)[name = tensor("cast_136")]; - tensor use_roll_7 = reshape(shape = var_630, x = var_629)[name = tensor("use_roll_7")]; - tensor var_632 = sub(x = var_17, y = use_roll_7)[name = tensor("op_632")]; - tensor var_633 = mul(x = filled_window_7, y = var_632)[name = tensor("op_633")]; - tensor var_634 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_634")]; - tensor input_81 = add(x = var_633, y = var_634)[name = tensor("input_81")]; - tensor var_636 = const()[name = tensor("op_636"), val = tensor(0x1p+0)]; - tensor var_637 = add(x = clip_5, y = var_636)[name = tensor("op_637")]; - tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_637)[name = tensor("clip_7")]; - tensor x_15_axes_0 = const()[name = tensor("x_15_axes_0"), val = tensor([-1])]; - tensor x_15 = layer_norm(axes = x_15_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_81)[name = tensor("x_15")]; - tensor input_83_perm_0 = const()[name = tensor("input_83_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_25_pad_type_0 = const()[name = tensor("inputs_25_pad_type_0"), val = tensor("valid")]; - tensor inputs_25_strides_0 = const()[name = tensor("inputs_25_strides_0"), val = tensor([1])]; - tensor inputs_25_pad_0 = const()[name = tensor("inputs_25_pad_0"), val = tensor([0, 0])]; - tensor inputs_25_dilations_0 = const()[name = tensor("inputs_25_dilations_0"), val = tensor([1])]; - tensor inputs_25_groups_0 = const()[name = tensor("inputs_25_groups_0"), val = tensor(1)]; - tensor input_83 = transpose(perm = input_83_perm_0, x = x_15)[name = tensor("transpose_48")]; - tensor inputs_25 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_25_dilations_0, groups = inputs_25_groups_0, pad = inputs_25_pad_0, pad_type = inputs_25_pad_type_0, strides = inputs_25_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_83)[name = tensor("inputs_25")]; - tensor var_658_split_sizes_0 = const()[name = tensor("op_658_split_sizes_0"), val = tensor([256, 256])]; - tensor var_658_axis_0 = const()[name = tensor("op_658_axis_0"), val = tensor(1)]; - tensor var_658_0, tensor var_658_1 = split(axis = var_658_axis_0, split_sizes = var_658_split_sizes_0, x = inputs_25)[name = tensor("op_658")]; - tensor var_660 = sigmoid(x = var_658_1)[name = tensor("op_660")]; - tensor inputs_27 = mul(x = var_658_0, y = var_660)[name = tensor("inputs_27")]; - tensor outputs_aug_7_pad_type_0 = const()[name = tensor("outputs_aug_7_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_7_pad_0 = const()[name = tensor("outputs_aug_7_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_7_groups_0 = const()[name = tensor("outputs_aug_7_groups_0"), val = tensor(256)]; - tensor outputs_aug_7_strides_0 = const()[name = tensor("outputs_aug_7_strides_0"), val = tensor([1])]; - tensor outputs_aug_7_dilations_0 = const()[name = tensor("outputs_aug_7_dilations_0"), val = tensor([1])]; - tensor outputs_aug_7 = conv(dilations = outputs_aug_7_dilations_0, groups = outputs_aug_7_groups_0, pad = outputs_aug_7_pad_0, pad_type = outputs_aug_7_pad_type_0, strides = outputs_aug_7_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_27)[name = tensor("outputs_aug_7")]; - tensor input_85_begin_0 = const()[name = tensor("input_85_begin_0"), val = tensor([0, 0, 0])]; - tensor input_85_end_0 = const()[name = tensor("input_85_end_0"), val = tensor([1, 256, 16])]; - tensor input_85_end_mask_0 = const()[name = tensor("input_85_end_mask_0"), val = tensor([true, true, false])]; - tensor input_85 = slice_by_index(begin = input_85_begin_0, end = input_85_end_0, end_mask = input_85_end_mask_0, x = outputs_aug_7)[name = tensor("input_85")]; - tensor inputs_29 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_85)[name = tensor("inputs_29")]; - tensor input_87 = silu(x = inputs_29)[name = tensor("input_87")]; - tensor input_89_pad_type_0 = const()[name = tensor("input_89_pad_type_0"), val = tensor("valid")]; - tensor input_89_strides_0 = const()[name = tensor("input_89_strides_0"), val = tensor([1])]; - tensor input_89_pad_0 = const()[name = tensor("input_89_pad_0"), val = tensor([0, 0])]; - tensor input_89_dilations_0 = const()[name = tensor("input_89_dilations_0"), val = tensor([1])]; - tensor input_89_groups_0 = const()[name = tensor("input_89_groups_0"), val = tensor(1)]; - tensor input_89 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_89_dilations_0, groups = input_89_groups_0, pad = input_89_pad_0, pad_type = input_89_pad_type_0, strides = input_89_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_87)[name = tensor("input_89")]; - tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_691_begin_0 = const()[name = tensor("op_691_begin_0"), val = tensor([0, -1, 0])]; - tensor var_691_end_0 = const()[name = tensor("op_691_end_0"), val = tensor([1, 16, 256])]; - tensor var_691_end_mask_0 = const()[name = tensor("op_691_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_89)[name = tensor("transpose_47")]; - tensor var_691 = slice_by_index(begin = var_691_begin_0, end = var_691_end_0, end_mask = var_691_end_mask_0, x = conv_out_7)[name = tensor("op_691")]; - tensor var_694_interleave_0 = const()[name = tensor("op_694_interleave_0"), val = tensor(false)]; - tensor var_694 = concat(axis = var_33, interleave = var_694_interleave_0, values = (var_607, var_691))[name = tensor("op_694")]; - tensor input_91 = add(x = x_11, y = var_694)[name = tensor("input_91")]; - tensor input_93_axes_0 = const()[name = tensor("input_93_axes_0"), val = tensor([-1])]; - tensor input_93 = layer_norm(axes = input_93_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_91)[name = tensor("input_93")]; - tensor inputs_31 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_93)[name = tensor("linear_17")]; - tensor input_95 = silu(x = inputs_31)[name = tensor("input_95")]; - tensor input_99 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_95)[name = tensor("linear_18")]; - tensor var_717 = const()[name = tensor("op_717"), val = tensor(0x1p-1)]; - tensor var_718 = mul(x = input_99, y = var_717)[name = tensor("op_718")]; - tensor input_101 = add(x = var_718, y = input_91)[name = tensor("input_101")]; - tensor input_103_axes_0 = const()[name = tensor("input_103_axes_0"), val = tensor([-1])]; - tensor input_103 = layer_norm(axes = input_103_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_101)[name = tensor("input_103")]; - tensor input_105_axes_0 = const()[name = tensor("input_105_axes_0"), val = tensor([-1])]; - tensor input_105 = layer_norm(axes = input_105_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_103)[name = tensor("input_105")]; - tensor inputs_33 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_105)[name = tensor("linear_19")]; - tensor input_107 = silu(x = inputs_33)[name = tensor("input_107")]; - tensor input_111 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_107)[name = tensor("linear_20")]; - tensor var_747 = const()[name = tensor("op_747"), val = tensor(0x1p-1)]; - tensor var_748 = mul(x = input_111, y = var_747)[name = tensor("op_748")]; - tensor input_113 = add(x = var_748, y = input_103)[name = tensor("input_113")]; - tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; - tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_113)[name = tensor("x_17")]; + tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_503)[name = tensor("x_expanded_7")]; + tensor var_517 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_517")]; + tensor var_518 = mul(x = window_9, y = var_517)[name = tensor("op_518")]; + tensor var_519 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_519")]; + tensor filled_window_7 = add(x = var_518, y = var_519)[name = tensor("filled_window_7")]; + tensor var_521 = greater_equal(x = clip_5, y = var_16)[name = tensor("op_521")]; + tensor var_522_dtype_0 = const()[name = tensor("op_522_dtype_0"), val = tensor("fp32")]; + tensor var_523 = const()[name = tensor("op_523"), val = tensor([1, 1, 1])]; + tensor var_522 = cast(dtype = var_522_dtype_0, x = var_521)[name = tensor("cast_136")]; + tensor use_roll_7 = reshape(shape = var_523, x = var_522)[name = tensor("use_roll_7")]; + tensor var_525 = sub(x = var_17, y = use_roll_7)[name = tensor("op_525")]; + tensor var_526 = mul(x = filled_window_7, y = var_525)[name = tensor("op_526")]; + tensor var_527 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_527")]; + tensor window_11 = add(x = var_526, y = var_527)[name = tensor("window_11")]; + tensor var_529 = const()[name = tensor("op_529"), val = tensor(0x1p+0)]; + tensor var_530 = add(x = clip_5, y = var_529)[name = tensor("op_530")]; + tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_530)[name = tensor("clip_7")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_30, interleave = input_61_interleave_0, values = (window_9, window_11))[name = tensor("input_61")]; + tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; + tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; + tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_13_pad_type_0 = const()[name = tensor("inputs_13_pad_type_0"), val = tensor("valid")]; + tensor inputs_13_strides_0 = const()[name = tensor("inputs_13_strides_0"), val = tensor([1])]; + tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; + tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; + tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; + tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; + tensor var_553_split_sizes_0 = const()[name = tensor("op_553_split_sizes_0"), val = tensor([256, 256])]; + tensor var_553_axis_0 = const()[name = tensor("op_553_axis_0"), val = tensor(1)]; + tensor var_553_0, tensor var_553_1 = split(axis = var_553_axis_0, split_sizes = var_553_split_sizes_0, x = inputs_13)[name = tensor("op_553")]; + tensor var_555 = sigmoid(x = var_553_1)[name = tensor("op_555")]; + tensor inputs_15 = mul(x = var_553_0, y = var_555)[name = tensor("inputs_15")]; + tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; + tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; + tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; + tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_15)[name = tensor("outputs_aug_3")]; + tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; + tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([2, 256, 16])]; + tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; + tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_3)[name = tensor("input_65")]; + tensor inputs_17 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_65)[name = tensor("inputs_17")]; + tensor input_67 = silu(x = inputs_17)[name = tensor("input_67")]; + tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; + tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; + tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; + tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; + tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; + tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; + tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; + tensor var_586_begin_0 = const()[name = tensor("op_586_begin_0"), val = tensor([0, -1, 0])]; + tensor var_586_end_0 = const()[name = tensor("op_586_end_0"), val = tensor([2, 16, 256])]; + tensor var_586_end_mask_0 = const()[name = tensor("op_586_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_586 = slice_by_index(begin = var_586_begin_0, end = var_586_end_0, end_mask = var_586_end_mask_0, x = conv_out_3)[name = tensor("op_586")]; + tensor var_588_perm_0 = const()[name = tensor("op_588_perm_0"), val = tensor([1, 0, 2])]; + tensor var_588 = transpose(perm = var_588_perm_0, x = var_586)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_588)[name = tensor("input_71")]; + tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; + tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; + tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; + tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; + tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; + tensor var_611 = const()[name = tensor("op_611"), val = tensor(0x1p-1)]; + tensor var_612 = mul(x = input_79, y = var_611)[name = tensor("op_612")]; + tensor input_81 = add(x = var_612, y = input_71)[name = tensor("input_81")]; + tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; + tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; + tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; + tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; + tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; + tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; + tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; + tensor var_641 = const()[name = tensor("op_641"), val = tensor(0x1p-1)]; + tensor var_642 = mul(x = input_91, y = var_641)[name = tensor("op_642")]; + tensor input_93 = add(x = var_642, y = input_83)[name = tensor("input_93")]; + tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; + tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; tensor prev_kv_5_end_0 = const()[name = tensor("prev_kv_5_end_0"), val = tensor([3, 1, 4, 64, 64])]; tensor prev_kv_5_end_mask_0 = const()[name = tensor("prev_kv_5_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -737,243 +664,206 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_762 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_17)[name = tensor("linear_21")]; - tensor var_763 = const()[name = tensor("op_763"), val = tensor([1, 2, 4, 64])]; - tensor var_764 = reshape(shape = var_763, x = var_762)[name = tensor("op_764")]; + tensor var_656 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_657 = const()[name = tensor("op_657"), val = tensor([1, 2, 4, 64])]; + tensor var_658 = reshape(shape = var_657, x = var_656)[name = tensor("op_658")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_768 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_17)[name = tensor("linear_22")]; - tensor var_769 = const()[name = tensor("op_769"), val = tensor(0x1p-3)]; - tensor var_770 = mul(x = var_768, y = var_769)[name = tensor("op_770")]; - tensor var_771 = const()[name = tensor("op_771"), val = tensor([1, 2, 4, 64])]; - tensor var_772 = reshape(shape = var_771, x = var_770)[name = tensor("op_772")]; + tensor var_662 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_663 = const()[name = tensor("op_663"), val = tensor(0x1p-3)]; + tensor var_664 = mul(x = var_662, y = var_663)[name = tensor("op_664")]; + tensor var_665 = const()[name = tensor("op_665"), val = tensor([1, 2, 4, 64])]; + tensor var_666 = reshape(shape = var_665, x = var_664)[name = tensor("op_666")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_776 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_17)[name = tensor("linear_23")]; - tensor var_777 = const()[name = tensor("op_777"), val = tensor([1, 2, 4, 64])]; - tensor var_778 = reshape(shape = var_777, x = var_776)[name = tensor("op_778")]; + tensor var_670 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_671 = const()[name = tensor("op_671"), val = tensor([1, 2, 4, 64])]; + tensor var_672 = reshape(shape = var_671, x = var_670)[name = tensor("op_672")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_117 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_17)[name = tensor("linear_24")]; + tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; tensor s_t_5 = add(x = prev_scale_5, y = encoder__t_index)[name = tensor("s_t_5")]; tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_772)[name = tensor("transpose_45")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_764)[name = tensor("transpose_46")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_666)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_658)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_788 = const()[name = tensor("op_788"), val = tensor([2, 1])]; - tensor var_789 = reshape(shape = var_788, x = sqrt_s_t_5)[name = tensor("op_789")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_789)[name = tensor("M_5")]; - tensor var_791 = mul(x = qk_5, y = M_5)[name = tensor("op_791")]; + tensor var_682 = const()[name = tensor("op_682"), val = tensor([2, 1])]; + tensor var_683 = reshape(shape = var_682, x = sqrt_s_t_5)[name = tensor("op_683")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_683)[name = tensor("M_5")]; + tensor var_685 = mul(x = qk_5, y = M_5)[name = tensor("op_685")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_778)[name = tensor("transpose_44")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_791, y = v_5)[name = tensor("inner_5")]; - tensor var_793_transpose_x_0 = const()[name = tensor("op_793_transpose_x_0"), val = tensor(false)]; - tensor var_793_transpose_y_0 = const()[name = tensor("op_793_transpose_y_0"), val = tensor(false)]; - tensor var_793 = matmul(transpose_x = var_793_transpose_x_0, transpose_y = var_793_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_793")]; - tensor var_794 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_794")]; - tensor var_795 = const()[name = tensor("op_795"), val = tensor([1, 1, 2, 1])]; - tensor var_796 = reshape(shape = var_795, x = var_794)[name = tensor("op_796")]; - tensor cross_5 = mul(x = var_793, y = var_796)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_672)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_685, y = v_5)[name = tensor("inner_5")]; + tensor var_687_transpose_x_0 = const()[name = tensor("op_687_transpose_x_0"), val = tensor(false)]; + tensor var_687_transpose_y_0 = const()[name = tensor("op_687_transpose_y_0"), val = tensor(false)]; + tensor var_687 = matmul(transpose_x = var_687_transpose_x_0, transpose_y = var_687_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_687")]; + tensor var_688 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_688")]; + tensor var_689 = const()[name = tensor("op_689"), val = tensor([1, 1, 2, 1])]; + tensor var_690 = reshape(shape = var_689, x = var_688)[name = tensor("op_690")]; + tensor cross_5 = mul(x = var_687, y = var_690)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_799 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_799")]; - tensor var_801_transpose_x_1 = const()[name = tensor("op_801_transpose_x_1"), val = tensor(true)]; - tensor var_801_transpose_y_1 = const()[name = tensor("op_801_transpose_y_1"), val = tensor(false)]; - tensor var_801 = matmul(transpose_x = var_801_transpose_x_1, transpose_y = var_801_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_801")]; - tensor new_kv_unnorm_5 = add(x = var_799, y = var_801)[name = tensor("new_kv_unnorm_5")]; - tensor var_803 = const()[name = tensor("op_803"), val = tensor(0x1p+1)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_803)[name = tensor("new_scale_5")]; - tensor var_805 = sqrt(x = new_scale_5)[name = tensor("op_805")]; - tensor var_806 = real_div(x = new_kv_unnorm_5, y = var_805)[name = tensor("op_806")]; - tensor var_807_perm_0 = const()[name = tensor("op_807_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_693 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_693")]; + tensor var_695_transpose_x_1 = const()[name = tensor("op_695_transpose_x_1"), val = tensor(true)]; + tensor var_695_transpose_y_1 = const()[name = tensor("op_695_transpose_y_1"), val = tensor(false)]; + tensor var_695 = matmul(transpose_x = var_695_transpose_x_1, transpose_y = var_695_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_695")]; + tensor new_kv_unnorm_5 = add(x = var_693, y = var_695)[name = tensor("new_kv_unnorm_5")]; + tensor var_697 = const()[name = tensor("op_697"), val = tensor(0x1p+1)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_697)[name = tensor("new_scale_5")]; + tensor var_699 = sqrt(x = new_scale_5)[name = tensor("op_699")]; + tensor var_700 = real_div(x = new_kv_unnorm_5, y = var_699)[name = tensor("op_700")]; + tensor var_701_perm_0 = const()[name = tensor("op_701_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_807 = transpose(perm = var_807_perm_0, x = out_13)[name = tensor("transpose_43")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_807)[name = tensor("out_15")]; - tensor var_811 = const()[name = tensor("op_811"), val = tensor([1, 2, 256])]; - tensor out_17 = reshape(shape = var_811, x = out_15)[name = tensor("out_17")]; - tensor var_813 = silu(x = input_117)[name = tensor("op_813")]; - tensor input_119 = mul(x = var_813, y = out_17)[name = tensor("input_119")]; - tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_119)[name = tensor("linear_25")]; - tensor x_19 = add(x = input_113, y = ret_out_5)[name = tensor("x_19")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor var_701 = transpose(perm = var_701_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_701)[name = tensor("out_15")]; + tensor var_705 = const()[name = tensor("op_705"), val = tensor([1, 2, 256])]; + tensor out_17 = reshape(shape = var_705, x = out_15)[name = tensor("out_17")]; + tensor var_707 = silu(x = input_97)[name = tensor("op_707")]; + tensor input_99 = mul(x = var_707, y = out_17)[name = tensor("input_99")]; + tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; + tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; + tensor window_13_begin_0 = const()[name = tensor("window_13_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_13_end_0 = const()[name = tensor("window_13_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_13_end_mask_0 = const()[name = tensor("window_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_13_squeeze_mask_0 = const()[name = tensor("window_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_13 = slice_by_index(begin = window_13_begin_0, end = window_13_end_0, end_mask = window_13_end_mask_0, squeeze_mask = window_13_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_13")]; tensor filled_9_begin_0 = const()[name = tensor("filled_9_begin_0"), val = tensor([2, 0])]; tensor filled_9_end_0 = const()[name = tensor("filled_9_end_0"), val = tensor([3, 1])]; tensor filled_9_end_mask_0 = const()[name = tensor("filled_9_end_mask_0"), val = tensor([false, true])]; tensor filled_9_squeeze_mask_0 = const()[name = tensor("filled_9_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_9 = slice_by_index(begin = filled_9_begin_0, end = filled_9_end_0, end_mask = filled_9_end_mask_0, squeeze_mask = filled_9_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_9")]; - tensor var_822_begin_0 = const()[name = tensor("op_822_begin_0"), val = tensor([0, 0, 0])]; - tensor var_822_end_0 = const()[name = tensor("op_822_end_0"), val = tensor([1, 1, 256])]; - tensor var_822_end_mask_0 = const()[name = tensor("op_822_end_mask_0"), val = tensor([true, false, true])]; - tensor var_822 = slice_by_index(begin = var_822_begin_0, end = var_822_end_0, end_mask = var_822_end_mask_0, x = x_19)[name = tensor("op_822")]; - tensor var_825_begin_0 = const()[name = tensor("op_825_begin_0"), val = tensor([0, 1, 0])]; - tensor var_825_end_0 = const()[name = tensor("op_825_end_0"), val = tensor([1, 16, 256])]; - tensor var_825_end_mask_0 = const()[name = tensor("op_825_end_mask_0"), val = tensor([true, true, true])]; - tensor var_825 = slice_by_index(begin = var_825_begin_0, end = var_825_end_0, end_mask = var_825_end_mask_0, x = window_5)[name = tensor("op_825")]; + tensor var_716_begin_0 = const()[name = tensor("op_716_begin_0"), val = tensor([0, 0, 0])]; + tensor var_716_end_0 = const()[name = tensor("op_716_end_0"), val = tensor([1, 1, 256])]; + tensor var_716_end_mask_0 = const()[name = tensor("op_716_end_mask_0"), val = tensor([true, false, true])]; + tensor var_716 = slice_by_index(begin = var_716_begin_0, end = var_716_end_0, end_mask = var_716_end_mask_0, x = x_15)[name = tensor("op_716")]; + tensor var_719_begin_0 = const()[name = tensor("op_719_begin_0"), val = tensor([0, 1, 0])]; + tensor var_719_end_0 = const()[name = tensor("op_719_end_0"), val = tensor([1, 16, 256])]; + tensor var_719_end_mask_0 = const()[name = tensor("op_719_end_mask_0"), val = tensor([true, true, true])]; + tensor var_719 = slice_by_index(begin = var_719_begin_0, end = var_719_end_0, end_mask = var_719_end_mask_0, x = window_13)[name = tensor("op_719")]; tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; - tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_825, var_822))[name = tensor("rolled_9")]; + tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_719, var_716))[name = tensor("rolled_9")]; tensor clip_8 = clip(alpha = var_21, beta = var_20, x = filled_9)[name = tensor("clip_8")]; - tensor var_830 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_830")]; - tensor var_831_dtype_0 = const()[name = tensor("op_831_dtype_0"), val = tensor("fp32")]; - tensor var_832 = const()[name = tensor("op_832"), val = tensor([1, 16, 1])]; - tensor var_831 = cast(dtype = var_831_dtype_0, x = var_830)[name = tensor("cast_135")]; - tensor fill_mask_9 = reshape(shape = var_832, x = var_831)[name = tensor("fill_mask_9")]; + tensor var_724 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_724")]; + tensor var_725_dtype_0 = const()[name = tensor("op_725_dtype_0"), val = tensor("fp32")]; + tensor var_726 = const()[name = tensor("op_726"), val = tensor([1, 16, 1])]; + tensor var_725 = cast(dtype = var_725_dtype_0, x = var_724)[name = tensor("cast_135")]; + tensor fill_mask_9 = reshape(shape = var_726, x = var_725)[name = tensor("fill_mask_9")]; tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_822)[name = tensor("x_expanded_9")]; - tensor var_836 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_836")]; - tensor var_837 = mul(x = window_5, y = var_836)[name = tensor("op_837")]; - tensor var_838 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_838")]; - tensor filled_window_9 = add(x = var_837, y = var_838)[name = tensor("filled_window_9")]; - tensor var_840 = greater_equal(x = filled_9, y = var_16)[name = tensor("op_840")]; - tensor var_841_dtype_0 = const()[name = tensor("op_841_dtype_0"), val = tensor("fp32")]; - tensor var_842 = const()[name = tensor("op_842"), val = tensor([1, 1, 1])]; - tensor var_841 = cast(dtype = var_841_dtype_0, x = var_840)[name = tensor("cast_134")]; - tensor use_roll_9 = reshape(shape = var_842, x = var_841)[name = tensor("use_roll_9")]; - tensor var_844 = sub(x = var_17, y = use_roll_9)[name = tensor("op_844")]; - tensor var_845 = mul(x = filled_window_9, y = var_844)[name = tensor("op_845")]; - tensor var_846 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_846")]; - tensor input_121 = add(x = var_845, y = var_846)[name = tensor("input_121")]; - tensor var_848 = const()[name = tensor("op_848"), val = tensor(0x1p+0)]; - tensor var_849 = add(x = filled_9, y = var_848)[name = tensor("op_849")]; - tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_849)[name = tensor("clip_9")]; - tensor x_21_axes_0 = const()[name = tensor("x_21_axes_0"), val = tensor([-1])]; - tensor x_21 = layer_norm(axes = x_21_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_121)[name = tensor("x_21")]; - tensor input_123_perm_0 = const()[name = tensor("input_123_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_35_pad_type_0 = const()[name = tensor("inputs_35_pad_type_0"), val = tensor("valid")]; - tensor inputs_35_strides_0 = const()[name = tensor("inputs_35_strides_0"), val = tensor([1])]; - tensor inputs_35_pad_0 = const()[name = tensor("inputs_35_pad_0"), val = tensor([0, 0])]; - tensor inputs_35_dilations_0 = const()[name = tensor("inputs_35_dilations_0"), val = tensor([1])]; - tensor inputs_35_groups_0 = const()[name = tensor("inputs_35_groups_0"), val = tensor(1)]; - tensor input_123 = transpose(perm = input_123_perm_0, x = x_21)[name = tensor("transpose_42")]; - tensor inputs_35 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_35_dilations_0, groups = inputs_35_groups_0, pad = inputs_35_pad_0, pad_type = inputs_35_pad_type_0, strides = inputs_35_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_123)[name = tensor("inputs_35")]; - tensor var_870_split_sizes_0 = const()[name = tensor("op_870_split_sizes_0"), val = tensor([256, 256])]; - tensor var_870_axis_0 = const()[name = tensor("op_870_axis_0"), val = tensor(1)]; - tensor var_870_0, tensor var_870_1 = split(axis = var_870_axis_0, split_sizes = var_870_split_sizes_0, x = inputs_35)[name = tensor("op_870")]; - tensor var_872 = sigmoid(x = var_870_1)[name = tensor("op_872")]; - tensor inputs_37 = mul(x = var_870_0, y = var_872)[name = tensor("inputs_37")]; - tensor outputs_aug_9_pad_type_0 = const()[name = tensor("outputs_aug_9_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_9_pad_0 = const()[name = tensor("outputs_aug_9_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_9_groups_0 = const()[name = tensor("outputs_aug_9_groups_0"), val = tensor(256)]; - tensor outputs_aug_9_strides_0 = const()[name = tensor("outputs_aug_9_strides_0"), val = tensor([1])]; - tensor outputs_aug_9_dilations_0 = const()[name = tensor("outputs_aug_9_dilations_0"), val = tensor([1])]; - tensor outputs_aug_9 = conv(dilations = outputs_aug_9_dilations_0, groups = outputs_aug_9_groups_0, pad = outputs_aug_9_pad_0, pad_type = outputs_aug_9_pad_type_0, strides = outputs_aug_9_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_37)[name = tensor("outputs_aug_9")]; - tensor input_125_begin_0 = const()[name = tensor("input_125_begin_0"), val = tensor([0, 0, 0])]; - tensor input_125_end_0 = const()[name = tensor("input_125_end_0"), val = tensor([1, 256, 16])]; - tensor input_125_end_mask_0 = const()[name = tensor("input_125_end_mask_0"), val = tensor([true, true, false])]; - tensor input_125 = slice_by_index(begin = input_125_begin_0, end = input_125_end_0, end_mask = input_125_end_mask_0, x = outputs_aug_9)[name = tensor("input_125")]; - tensor inputs_39 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_125)[name = tensor("inputs_39")]; - tensor input_127 = silu(x = inputs_39)[name = tensor("input_127")]; - tensor input_129_pad_type_0 = const()[name = tensor("input_129_pad_type_0"), val = tensor("valid")]; - tensor input_129_strides_0 = const()[name = tensor("input_129_strides_0"), val = tensor([1])]; - tensor input_129_pad_0 = const()[name = tensor("input_129_pad_0"), val = tensor([0, 0])]; - tensor input_129_dilations_0 = const()[name = tensor("input_129_dilations_0"), val = tensor([1])]; - tensor input_129_groups_0 = const()[name = tensor("input_129_groups_0"), val = tensor(1)]; - tensor input_129 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_129_dilations_0, groups = input_129_groups_0, pad = input_129_pad_0, pad_type = input_129_pad_type_0, strides = input_129_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_127)[name = tensor("input_129")]; - tensor conv_out_9_perm_0 = const()[name = tensor("conv_out_9_perm_0"), val = tensor([0, 2, 1])]; - tensor var_903_begin_0 = const()[name = tensor("op_903_begin_0"), val = tensor([0, -1, 0])]; - tensor var_903_end_0 = const()[name = tensor("op_903_end_0"), val = tensor([1, 16, 256])]; - tensor var_903_end_mask_0 = const()[name = tensor("op_903_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_9 = transpose(perm = conv_out_9_perm_0, x = input_129)[name = tensor("transpose_41")]; - tensor var_903 = slice_by_index(begin = var_903_begin_0, end = var_903_end_0, end_mask = var_903_end_mask_0, x = conv_out_9)[name = tensor("op_903")]; - tensor var_906_begin_0 = const()[name = tensor("op_906_begin_0"), val = tensor([0, 1, 0])]; - tensor var_906_end_0 = const()[name = tensor("op_906_end_0"), val = tensor([1, 1, 256])]; - tensor var_906_end_mask_0 = const()[name = tensor("op_906_end_mask_0"), val = tensor([true, true, true])]; - tensor var_906 = slice_by_index(begin = var_906_begin_0, end = var_906_end_0, end_mask = var_906_end_mask_0, x = x_19)[name = tensor("op_906")]; - tensor var_909_begin_0 = const()[name = tensor("op_909_begin_0"), val = tensor([0, 1, 0])]; - tensor var_909_end_0 = const()[name = tensor("op_909_end_0"), val = tensor([1, 16, 256])]; - tensor var_909_end_mask_0 = const()[name = tensor("op_909_end_mask_0"), val = tensor([true, true, true])]; - tensor var_909 = slice_by_index(begin = var_909_begin_0, end = var_909_end_0, end_mask = var_909_end_mask_0, x = input_121)[name = tensor("op_909")]; + tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_716)[name = tensor("x_expanded_9")]; + tensor var_730 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_730")]; + tensor var_731 = mul(x = window_13, y = var_730)[name = tensor("op_731")]; + tensor var_732 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_732")]; + tensor filled_window_9 = add(x = var_731, y = var_732)[name = tensor("filled_window_9")]; + tensor var_734 = greater_equal(x = filled_9, y = var_16)[name = tensor("op_734")]; + tensor var_735_dtype_0 = const()[name = tensor("op_735_dtype_0"), val = tensor("fp32")]; + tensor var_736 = const()[name = tensor("op_736"), val = tensor([1, 1, 1])]; + tensor var_735 = cast(dtype = var_735_dtype_0, x = var_734)[name = tensor("cast_134")]; + tensor use_roll_9 = reshape(shape = var_736, x = var_735)[name = tensor("use_roll_9")]; + tensor var_738 = sub(x = var_17, y = use_roll_9)[name = tensor("op_738")]; + tensor var_739 = mul(x = filled_window_9, y = var_738)[name = tensor("op_739")]; + tensor var_740 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_740")]; + tensor window_15 = add(x = var_739, y = var_740)[name = tensor("window_15")]; + tensor var_742 = const()[name = tensor("op_742"), val = tensor(0x1p+0)]; + tensor var_743 = add(x = filled_9, y = var_742)[name = tensor("op_743")]; + tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_743)[name = tensor("clip_9")]; + tensor var_746_begin_0 = const()[name = tensor("op_746_begin_0"), val = tensor([0, 1, 0])]; + tensor var_746_end_0 = const()[name = tensor("op_746_end_0"), val = tensor([1, 1, 256])]; + tensor var_746_end_mask_0 = const()[name = tensor("op_746_end_mask_0"), val = tensor([true, true, true])]; + tensor var_746 = slice_by_index(begin = var_746_begin_0, end = var_746_end_0, end_mask = var_746_end_mask_0, x = x_15)[name = tensor("op_746")]; + tensor var_749_begin_0 = const()[name = tensor("op_749_begin_0"), val = tensor([0, 1, 0])]; + tensor var_749_end_0 = const()[name = tensor("op_749_end_0"), val = tensor([1, 16, 256])]; + tensor var_749_end_mask_0 = const()[name = tensor("op_749_end_mask_0"), val = tensor([true, true, true])]; + tensor var_749 = slice_by_index(begin = var_749_begin_0, end = var_749_end_0, end_mask = var_749_end_mask_0, x = window_15)[name = tensor("op_749")]; tensor rolled_11_interleave_0 = const()[name = tensor("rolled_11_interleave_0"), val = tensor(false)]; - tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_909, var_906))[name = tensor("rolled_11")]; + tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_749, var_746))[name = tensor("rolled_11")]; tensor clip_10 = clip(alpha = var_21, beta = var_20, x = clip_9)[name = tensor("clip_10")]; - tensor var_914 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_914")]; - tensor var_915_dtype_0 = const()[name = tensor("op_915_dtype_0"), val = tensor("fp32")]; - tensor var_916 = const()[name = tensor("op_916"), val = tensor([1, 16, 1])]; - tensor var_915 = cast(dtype = var_915_dtype_0, x = var_914)[name = tensor("cast_133")]; - tensor fill_mask_11 = reshape(shape = var_916, x = var_915)[name = tensor("fill_mask_11")]; + tensor var_754 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_754")]; + tensor var_755_dtype_0 = const()[name = tensor("op_755_dtype_0"), val = tensor("fp32")]; + tensor var_756 = const()[name = tensor("op_756"), val = tensor([1, 16, 1])]; + tensor var_755 = cast(dtype = var_755_dtype_0, x = var_754)[name = tensor("cast_133")]; + tensor fill_mask_11 = reshape(shape = var_756, x = var_755)[name = tensor("fill_mask_11")]; tensor x_expanded_11_reps_0 = const()[name = tensor("x_expanded_11_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_906)[name = tensor("x_expanded_11")]; - tensor var_920 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_920")]; - tensor var_921 = mul(x = input_121, y = var_920)[name = tensor("op_921")]; - tensor var_922 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_922")]; - tensor filled_window_11 = add(x = var_921, y = var_922)[name = tensor("filled_window_11")]; - tensor var_924 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_924")]; - tensor var_925_dtype_0 = const()[name = tensor("op_925_dtype_0"), val = tensor("fp32")]; - tensor var_926 = const()[name = tensor("op_926"), val = tensor([1, 1, 1])]; - tensor var_925 = cast(dtype = var_925_dtype_0, x = var_924)[name = tensor("cast_132")]; - tensor use_roll_11 = reshape(shape = var_926, x = var_925)[name = tensor("use_roll_11")]; - tensor var_928 = sub(x = var_17, y = use_roll_11)[name = tensor("op_928")]; - tensor var_929 = mul(x = filled_window_11, y = var_928)[name = tensor("op_929")]; - tensor var_930 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_930")]; - tensor input_131 = add(x = var_929, y = var_930)[name = tensor("input_131")]; - tensor var_932 = const()[name = tensor("op_932"), val = tensor(0x1p+0)]; - tensor var_933 = add(x = clip_9, y = var_932)[name = tensor("op_933")]; - tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_933)[name = tensor("clip_11")]; - tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; - tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_131)[name = tensor("x_23")]; - tensor input_133_perm_0 = const()[name = tensor("input_133_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_41_pad_type_0 = const()[name = tensor("inputs_41_pad_type_0"), val = tensor("valid")]; - tensor inputs_41_strides_0 = const()[name = tensor("inputs_41_strides_0"), val = tensor([1])]; - tensor inputs_41_pad_0 = const()[name = tensor("inputs_41_pad_0"), val = tensor([0, 0])]; - tensor inputs_41_dilations_0 = const()[name = tensor("inputs_41_dilations_0"), val = tensor([1])]; - tensor inputs_41_groups_0 = const()[name = tensor("inputs_41_groups_0"), val = tensor(1)]; - tensor input_133 = transpose(perm = input_133_perm_0, x = x_23)[name = tensor("transpose_40")]; - tensor inputs_41 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_41_dilations_0, groups = inputs_41_groups_0, pad = inputs_41_pad_0, pad_type = inputs_41_pad_type_0, strides = inputs_41_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_133)[name = tensor("inputs_41")]; - tensor var_954_split_sizes_0 = const()[name = tensor("op_954_split_sizes_0"), val = tensor([256, 256])]; - tensor var_954_axis_0 = const()[name = tensor("op_954_axis_0"), val = tensor(1)]; - tensor var_954_0, tensor var_954_1 = split(axis = var_954_axis_0, split_sizes = var_954_split_sizes_0, x = inputs_41)[name = tensor("op_954")]; - tensor var_956 = sigmoid(x = var_954_1)[name = tensor("op_956")]; - tensor inputs_43 = mul(x = var_954_0, y = var_956)[name = tensor("inputs_43")]; - tensor outputs_aug_11_pad_type_0 = const()[name = tensor("outputs_aug_11_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_11_pad_0 = const()[name = tensor("outputs_aug_11_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_11_groups_0 = const()[name = tensor("outputs_aug_11_groups_0"), val = tensor(256)]; - tensor outputs_aug_11_strides_0 = const()[name = tensor("outputs_aug_11_strides_0"), val = tensor([1])]; - tensor outputs_aug_11_dilations_0 = const()[name = tensor("outputs_aug_11_dilations_0"), val = tensor([1])]; - tensor outputs_aug_11 = conv(dilations = outputs_aug_11_dilations_0, groups = outputs_aug_11_groups_0, pad = outputs_aug_11_pad_0, pad_type = outputs_aug_11_pad_type_0, strides = outputs_aug_11_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_43)[name = tensor("outputs_aug_11")]; - tensor input_135_begin_0 = const()[name = tensor("input_135_begin_0"), val = tensor([0, 0, 0])]; - tensor input_135_end_0 = const()[name = tensor("input_135_end_0"), val = tensor([1, 256, 16])]; - tensor input_135_end_mask_0 = const()[name = tensor("input_135_end_mask_0"), val = tensor([true, true, false])]; - tensor input_135 = slice_by_index(begin = input_135_begin_0, end = input_135_end_0, end_mask = input_135_end_mask_0, x = outputs_aug_11)[name = tensor("input_135")]; - tensor inputs_45 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_135)[name = tensor("inputs_45")]; - tensor input_137 = silu(x = inputs_45)[name = tensor("input_137")]; - tensor input_139_pad_type_0 = const()[name = tensor("input_139_pad_type_0"), val = tensor("valid")]; - tensor input_139_strides_0 = const()[name = tensor("input_139_strides_0"), val = tensor([1])]; - tensor input_139_pad_0 = const()[name = tensor("input_139_pad_0"), val = tensor([0, 0])]; - tensor input_139_dilations_0 = const()[name = tensor("input_139_dilations_0"), val = tensor([1])]; - tensor input_139_groups_0 = const()[name = tensor("input_139_groups_0"), val = tensor(1)]; - tensor input_139 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_139_dilations_0, groups = input_139_groups_0, pad = input_139_pad_0, pad_type = input_139_pad_type_0, strides = input_139_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_137)[name = tensor("input_139")]; - tensor conv_out_11_perm_0 = const()[name = tensor("conv_out_11_perm_0"), val = tensor([0, 2, 1])]; - tensor var_987_begin_0 = const()[name = tensor("op_987_begin_0"), val = tensor([0, -1, 0])]; - tensor var_987_end_0 = const()[name = tensor("op_987_end_0"), val = tensor([1, 16, 256])]; - tensor var_987_end_mask_0 = const()[name = tensor("op_987_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_11 = transpose(perm = conv_out_11_perm_0, x = input_139)[name = tensor("transpose_39")]; - tensor var_987 = slice_by_index(begin = var_987_begin_0, end = var_987_end_0, end_mask = var_987_end_mask_0, x = conv_out_11)[name = tensor("op_987")]; - tensor var_990_interleave_0 = const()[name = tensor("op_990_interleave_0"), val = tensor(false)]; - tensor var_990 = concat(axis = var_33, interleave = var_990_interleave_0, values = (var_903, var_987))[name = tensor("op_990")]; - tensor input_141 = add(x = x_19, y = var_990)[name = tensor("input_141")]; - tensor input_143_axes_0 = const()[name = tensor("input_143_axes_0"), val = tensor([-1])]; - tensor input_143 = layer_norm(axes = input_143_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_141)[name = tensor("input_143")]; - tensor inputs_47 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_143)[name = tensor("linear_26")]; - tensor input_145 = silu(x = inputs_47)[name = tensor("input_145")]; - tensor input_149 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_145)[name = tensor("linear_27")]; - tensor var_1013 = const()[name = tensor("op_1013"), val = tensor(0x1p-1)]; - tensor var_1014 = mul(x = input_149, y = var_1013)[name = tensor("op_1014")]; - tensor input_151 = add(x = var_1014, y = input_141)[name = tensor("input_151")]; - tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; - tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_151)[name = tensor("input_153")]; - tensor input_155_axes_0 = const()[name = tensor("input_155_axes_0"), val = tensor([-1])]; - tensor input_155 = layer_norm(axes = input_155_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_153)[name = tensor("input_155")]; - tensor inputs_49 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_155)[name = tensor("linear_28")]; - tensor input_157 = silu(x = inputs_49)[name = tensor("input_157")]; - tensor input_161 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_157)[name = tensor("linear_29")]; - tensor var_1043 = const()[name = tensor("op_1043"), val = tensor(0x1p-1)]; - tensor var_1044 = mul(x = input_161, y = var_1043)[name = tensor("op_1044")]; - tensor input_163 = add(x = var_1044, y = input_153)[name = tensor("input_163")]; - tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; - tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_163)[name = tensor("x_25")]; + tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_746)[name = tensor("x_expanded_11")]; + tensor var_760 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_760")]; + tensor var_761 = mul(x = window_15, y = var_760)[name = tensor("op_761")]; + tensor var_762 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_762")]; + tensor filled_window_11 = add(x = var_761, y = var_762)[name = tensor("filled_window_11")]; + tensor var_764 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_764")]; + tensor var_765_dtype_0 = const()[name = tensor("op_765_dtype_0"), val = tensor("fp32")]; + tensor var_766 = const()[name = tensor("op_766"), val = tensor([1, 1, 1])]; + tensor var_765 = cast(dtype = var_765_dtype_0, x = var_764)[name = tensor("cast_132")]; + tensor use_roll_11 = reshape(shape = var_766, x = var_765)[name = tensor("use_roll_11")]; + tensor var_768 = sub(x = var_17, y = use_roll_11)[name = tensor("op_768")]; + tensor var_769 = mul(x = filled_window_11, y = var_768)[name = tensor("op_769")]; + tensor var_770 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_770")]; + tensor window_17 = add(x = var_769, y = var_770)[name = tensor("window_17")]; + tensor var_772 = const()[name = tensor("op_772"), val = tensor(0x1p+0)]; + tensor var_773 = add(x = clip_9, y = var_772)[name = tensor("op_773")]; + tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_773)[name = tensor("clip_11")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_30, interleave = input_101_interleave_0, values = (window_15, window_17))[name = tensor("input_101")]; + tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; + tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; + tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_23_pad_type_0 = const()[name = tensor("inputs_23_pad_type_0"), val = tensor("valid")]; + tensor inputs_23_strides_0 = const()[name = tensor("inputs_23_strides_0"), val = tensor([1])]; + tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; + tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; + tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; + tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; + tensor var_796_split_sizes_0 = const()[name = tensor("op_796_split_sizes_0"), val = tensor([256, 256])]; + tensor var_796_axis_0 = const()[name = tensor("op_796_axis_0"), val = tensor(1)]; + tensor var_796_0, tensor var_796_1 = split(axis = var_796_axis_0, split_sizes = var_796_split_sizes_0, x = inputs_23)[name = tensor("op_796")]; + tensor var_798 = sigmoid(x = var_796_1)[name = tensor("op_798")]; + tensor inputs_25 = mul(x = var_796_0, y = var_798)[name = tensor("inputs_25")]; + tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; + tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; + tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; + tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_25)[name = tensor("outputs_aug_5")]; + tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; + tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([2, 256, 16])]; + tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; + tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_5)[name = tensor("input_105")]; + tensor inputs_27 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_105)[name = tensor("inputs_27")]; + tensor input_107 = silu(x = inputs_27)[name = tensor("input_107")]; + tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; + tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; + tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; + tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; + tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; + tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; + tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; + tensor var_829_begin_0 = const()[name = tensor("op_829_begin_0"), val = tensor([0, -1, 0])]; + tensor var_829_end_0 = const()[name = tensor("op_829_end_0"), val = tensor([2, 16, 256])]; + tensor var_829_end_mask_0 = const()[name = tensor("op_829_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_829 = slice_by_index(begin = var_829_begin_0, end = var_829_end_0, end_mask = var_829_end_mask_0, x = conv_out_5)[name = tensor("op_829")]; + tensor var_831_perm_0 = const()[name = tensor("op_831_perm_0"), val = tensor([1, 0, 2])]; + tensor var_831 = transpose(perm = var_831_perm_0, x = var_829)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_831)[name = tensor("input_111")]; + tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; + tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; + tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; + tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; + tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; + tensor var_854 = const()[name = tensor("op_854"), val = tensor(0x1p-1)]; + tensor var_855 = mul(x = input_119, y = var_854)[name = tensor("op_855")]; + tensor input_121 = add(x = var_855, y = input_111)[name = tensor("input_121")]; + tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; + tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; + tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; + tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; + tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; + tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; + tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; + tensor var_884 = const()[name = tensor("op_884"), val = tensor(0x1p-1)]; + tensor var_885 = mul(x = input_131, y = var_884)[name = tensor("op_885")]; + tensor input_133 = add(x = var_885, y = input_123)[name = tensor("input_133")]; + tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; + tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; tensor prev_kv_7_end_0 = const()[name = tensor("prev_kv_7_end_0"), val = tensor([4, 1, 4, 64, 64])]; tensor prev_kv_7_end_mask_0 = const()[name = tensor("prev_kv_7_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -984,236 +874,199 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_1058 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_25)[name = tensor("linear_30")]; - tensor var_1059 = const()[name = tensor("op_1059"), val = tensor([1, 2, 4, 64])]; - tensor var_1060 = reshape(shape = var_1059, x = var_1058)[name = tensor("op_1060")]; + tensor var_899 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_900 = const()[name = tensor("op_900"), val = tensor([1, 2, 4, 64])]; + tensor var_901 = reshape(shape = var_900, x = var_899)[name = tensor("op_901")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1064 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_25)[name = tensor("linear_31")]; - tensor var_1065 = const()[name = tensor("op_1065"), val = tensor(0x1p-3)]; - tensor var_1066 = mul(x = var_1064, y = var_1065)[name = tensor("op_1066")]; - tensor var_1067 = const()[name = tensor("op_1067"), val = tensor([1, 2, 4, 64])]; - tensor var_1068 = reshape(shape = var_1067, x = var_1066)[name = tensor("op_1068")]; + tensor var_905 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_906 = const()[name = tensor("op_906"), val = tensor(0x1p-3)]; + tensor var_907 = mul(x = var_905, y = var_906)[name = tensor("op_907")]; + tensor var_908 = const()[name = tensor("op_908"), val = tensor([1, 2, 4, 64])]; + tensor var_909 = reshape(shape = var_908, x = var_907)[name = tensor("op_909")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1072 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_25)[name = tensor("linear_32")]; - tensor var_1073 = const()[name = tensor("op_1073"), val = tensor([1, 2, 4, 64])]; - tensor var_1074 = reshape(shape = var_1073, x = var_1072)[name = tensor("op_1074")]; + tensor var_913 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_914 = const()[name = tensor("op_914"), val = tensor([1, 2, 4, 64])]; + tensor var_915 = reshape(shape = var_914, x = var_913)[name = tensor("op_915")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_167 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_25)[name = tensor("linear_33")]; + tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; tensor s_t_7 = add(x = prev_scale_7, y = encoder__t_index)[name = tensor("s_t_7")]; tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_1068)[name = tensor("transpose_37")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_1060)[name = tensor("transpose_38")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_909)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_901)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_1084 = const()[name = tensor("op_1084"), val = tensor([2, 1])]; - tensor var_1085 = reshape(shape = var_1084, x = sqrt_s_t_7)[name = tensor("op_1085")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_1085)[name = tensor("M_7")]; - tensor var_1087 = mul(x = qk_7, y = M_7)[name = tensor("op_1087")]; + tensor var_925 = const()[name = tensor("op_925"), val = tensor([2, 1])]; + tensor var_926 = reshape(shape = var_925, x = sqrt_s_t_7)[name = tensor("op_926")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_926)[name = tensor("M_7")]; + tensor var_928 = mul(x = qk_7, y = M_7)[name = tensor("op_928")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_1074)[name = tensor("transpose_36")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1087, y = v_7)[name = tensor("inner_7")]; - tensor var_1089_transpose_x_0 = const()[name = tensor("op_1089_transpose_x_0"), val = tensor(false)]; - tensor var_1089_transpose_y_0 = const()[name = tensor("op_1089_transpose_y_0"), val = tensor(false)]; - tensor var_1089 = matmul(transpose_x = var_1089_transpose_x_0, transpose_y = var_1089_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1089")]; - tensor var_1090 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1090")]; - tensor var_1091 = const()[name = tensor("op_1091"), val = tensor([1, 1, 2, 1])]; - tensor var_1092 = reshape(shape = var_1091, x = var_1090)[name = tensor("op_1092")]; - tensor cross_7 = mul(x = var_1089, y = var_1092)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_915)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_928, y = v_7)[name = tensor("inner_7")]; + tensor var_930_transpose_x_0 = const()[name = tensor("op_930_transpose_x_0"), val = tensor(false)]; + tensor var_930_transpose_y_0 = const()[name = tensor("op_930_transpose_y_0"), val = tensor(false)]; + tensor var_930 = matmul(transpose_x = var_930_transpose_x_0, transpose_y = var_930_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_930")]; + tensor var_931 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_931")]; + tensor var_932 = const()[name = tensor("op_932"), val = tensor([1, 1, 2, 1])]; + tensor var_933 = reshape(shape = var_932, x = var_931)[name = tensor("op_933")]; + tensor cross_7 = mul(x = var_930, y = var_933)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_1095 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1095")]; - tensor var_1097_transpose_x_1 = const()[name = tensor("op_1097_transpose_x_1"), val = tensor(true)]; - tensor var_1097_transpose_y_1 = const()[name = tensor("op_1097_transpose_y_1"), val = tensor(false)]; - tensor var_1097 = matmul(transpose_x = var_1097_transpose_x_1, transpose_y = var_1097_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1097")]; - tensor new_kv_unnorm_7 = add(x = var_1095, y = var_1097)[name = tensor("new_kv_unnorm_7")]; - tensor var_1099 = const()[name = tensor("op_1099"), val = tensor(0x1p+1)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_1099)[name = tensor("new_scale_7")]; - tensor var_1101 = sqrt(x = new_scale_7)[name = tensor("op_1101")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1101)[name = tensor("nkv_1")]; - tensor var_1103_perm_0 = const()[name = tensor("op_1103_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_936 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_936")]; + tensor var_938_transpose_x_1 = const()[name = tensor("op_938_transpose_x_1"), val = tensor(true)]; + tensor var_938_transpose_y_1 = const()[name = tensor("op_938_transpose_y_1"), val = tensor(false)]; + tensor var_938 = matmul(transpose_x = var_938_transpose_x_1, transpose_y = var_938_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_938")]; + tensor new_kv_unnorm_7 = add(x = var_936, y = var_938)[name = tensor("new_kv_unnorm_7")]; + tensor var_940 = const()[name = tensor("op_940"), val = tensor(0x1p+1)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_940)[name = tensor("new_scale_7")]; + tensor var_942 = sqrt(x = new_scale_7)[name = tensor("op_942")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_942)[name = tensor("nkv_1")]; + tensor var_944_perm_0 = const()[name = tensor("op_944_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_1103 = transpose(perm = var_1103_perm_0, x = out_19)[name = tensor("transpose_35")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1103)[name = tensor("out_21")]; - tensor var_1107 = const()[name = tensor("op_1107"), val = tensor([1, 2, 256])]; - tensor out_23 = reshape(shape = var_1107, x = out_21)[name = tensor("out_23")]; - tensor var_1109 = silu(x = input_167)[name = tensor("op_1109")]; - tensor input_169 = mul(x = var_1109, y = out_23)[name = tensor("input_169")]; - tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_169)[name = tensor("linear_34")]; - tensor x_27 = add(x = input_163, y = ret_out_7)[name = tensor("x_27")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor var_944 = transpose(perm = var_944_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_944)[name = tensor("out_21")]; + tensor var_948 = const()[name = tensor("op_948"), val = tensor([1, 2, 256])]; + tensor out_23 = reshape(shape = var_948, x = out_21)[name = tensor("out_23")]; + tensor var_950 = silu(x = input_137)[name = tensor("op_950")]; + tensor input_139 = mul(x = var_950, y = out_23)[name = tensor("input_139")]; + tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; + tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; + tensor window_19_begin_0 = const()[name = tensor("window_19_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_19_end_0 = const()[name = tensor("window_19_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_19_end_mask_0 = const()[name = tensor("window_19_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_19_squeeze_mask_0 = const()[name = tensor("window_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_19 = slice_by_index(begin = window_19_begin_0, end = window_19_end_0, end_mask = window_19_end_mask_0, squeeze_mask = window_19_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_19")]; tensor filled_13_begin_0 = const()[name = tensor("filled_13_begin_0"), val = tensor([3, 0])]; tensor filled_13_end_0 = const()[name = tensor("filled_13_end_0"), val = tensor([4, 1])]; tensor filled_13_end_mask_0 = const()[name = tensor("filled_13_end_mask_0"), val = tensor([false, true])]; tensor filled_13_squeeze_mask_0 = const()[name = tensor("filled_13_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_13 = slice_by_index(begin = filled_13_begin_0, end = filled_13_end_0, end_mask = filled_13_end_mask_0, squeeze_mask = filled_13_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_13")]; - tensor var_1118_begin_0 = const()[name = tensor("op_1118_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1118_end_0 = const()[name = tensor("op_1118_end_0"), val = tensor([1, 1, 256])]; - tensor var_1118_end_mask_0 = const()[name = tensor("op_1118_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1118 = slice_by_index(begin = var_1118_begin_0, end = var_1118_end_0, end_mask = var_1118_end_mask_0, x = x_27)[name = tensor("op_1118")]; - tensor var_1121_begin_0 = const()[name = tensor("op_1121_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1121_end_0 = const()[name = tensor("op_1121_end_0"), val = tensor([1, 16, 256])]; - tensor var_1121_end_mask_0 = const()[name = tensor("op_1121_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1121 = slice_by_index(begin = var_1121_begin_0, end = var_1121_end_0, end_mask = var_1121_end_mask_0, x = window)[name = tensor("op_1121")]; + tensor var_959_begin_0 = const()[name = tensor("op_959_begin_0"), val = tensor([0, 0, 0])]; + tensor var_959_end_0 = const()[name = tensor("op_959_end_0"), val = tensor([1, 1, 256])]; + tensor var_959_end_mask_0 = const()[name = tensor("op_959_end_mask_0"), val = tensor([true, false, true])]; + tensor var_959 = slice_by_index(begin = var_959_begin_0, end = var_959_end_0, end_mask = var_959_end_mask_0, x = x_21)[name = tensor("op_959")]; + tensor var_962_begin_0 = const()[name = tensor("op_962_begin_0"), val = tensor([0, 1, 0])]; + tensor var_962_end_0 = const()[name = tensor("op_962_end_0"), val = tensor([1, 16, 256])]; + tensor var_962_end_mask_0 = const()[name = tensor("op_962_end_mask_0"), val = tensor([true, true, true])]; + tensor var_962 = slice_by_index(begin = var_962_begin_0, end = var_962_end_0, end_mask = var_962_end_mask_0, x = window_19)[name = tensor("op_962")]; tensor rolled_13_interleave_0 = const()[name = tensor("rolled_13_interleave_0"), val = tensor(false)]; - tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_1121, var_1118))[name = tensor("rolled_13")]; + tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_962, var_959))[name = tensor("rolled_13")]; tensor clip_12 = clip(alpha = var_21, beta = var_20, x = filled_13)[name = tensor("clip_12")]; - tensor var_1126 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_1126")]; - tensor var_1127_dtype_0 = const()[name = tensor("op_1127_dtype_0"), val = tensor("fp32")]; - tensor var_1128 = const()[name = tensor("op_1128"), val = tensor([1, 16, 1])]; - tensor var_1127 = cast(dtype = var_1127_dtype_0, x = var_1126)[name = tensor("cast_131")]; - tensor fill_mask_13 = reshape(shape = var_1128, x = var_1127)[name = tensor("fill_mask_13")]; + tensor var_967 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_967")]; + tensor var_968_dtype_0 = const()[name = tensor("op_968_dtype_0"), val = tensor("fp32")]; + tensor var_969 = const()[name = tensor("op_969"), val = tensor([1, 16, 1])]; + tensor var_968 = cast(dtype = var_968_dtype_0, x = var_967)[name = tensor("cast_131")]; + tensor fill_mask_13 = reshape(shape = var_969, x = var_968)[name = tensor("fill_mask_13")]; tensor x_expanded_13_reps_0 = const()[name = tensor("x_expanded_13_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_1118)[name = tensor("x_expanded_13")]; - tensor var_1132 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_1132")]; - tensor var_1133 = mul(x = window, y = var_1132)[name = tensor("op_1133")]; - tensor var_1134 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_1134")]; - tensor filled_window_13 = add(x = var_1133, y = var_1134)[name = tensor("filled_window_13")]; - tensor var_1136 = greater_equal(x = filled_13, y = var_16)[name = tensor("op_1136")]; - tensor var_1137_dtype_0 = const()[name = tensor("op_1137_dtype_0"), val = tensor("fp32")]; - tensor var_1138 = const()[name = tensor("op_1138"), val = tensor([1, 1, 1])]; - tensor var_1137 = cast(dtype = var_1137_dtype_0, x = var_1136)[name = tensor("cast_130")]; - tensor use_roll_13 = reshape(shape = var_1138, x = var_1137)[name = tensor("use_roll_13")]; - tensor var_1140 = sub(x = var_17, y = use_roll_13)[name = tensor("op_1140")]; - tensor var_1141 = mul(x = filled_window_13, y = var_1140)[name = tensor("op_1141")]; - tensor var_1142 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_1142")]; - tensor input_171 = add(x = var_1141, y = var_1142)[name = tensor("input_171")]; - tensor var_1144 = const()[name = tensor("op_1144"), val = tensor(0x1p+0)]; - tensor var_1145 = add(x = filled_13, y = var_1144)[name = tensor("op_1145")]; - tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_1145)[name = tensor("clip_13")]; - tensor x_29_axes_0 = const()[name = tensor("x_29_axes_0"), val = tensor([-1])]; - tensor x_29 = layer_norm(axes = x_29_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_171)[name = tensor("x_29")]; - tensor input_173_perm_0 = const()[name = tensor("input_173_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_51_pad_type_0 = const()[name = tensor("inputs_51_pad_type_0"), val = tensor("valid")]; - tensor inputs_51_strides_0 = const()[name = tensor("inputs_51_strides_0"), val = tensor([1])]; - tensor inputs_51_pad_0 = const()[name = tensor("inputs_51_pad_0"), val = tensor([0, 0])]; - tensor inputs_51_dilations_0 = const()[name = tensor("inputs_51_dilations_0"), val = tensor([1])]; - tensor inputs_51_groups_0 = const()[name = tensor("inputs_51_groups_0"), val = tensor(1)]; - tensor input_173 = transpose(perm = input_173_perm_0, x = x_29)[name = tensor("transpose_34")]; - tensor inputs_51 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_51_dilations_0, groups = inputs_51_groups_0, pad = inputs_51_pad_0, pad_type = inputs_51_pad_type_0, strides = inputs_51_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_173)[name = tensor("inputs_51")]; - tensor var_1166_split_sizes_0 = const()[name = tensor("op_1166_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1166_axis_0 = const()[name = tensor("op_1166_axis_0"), val = tensor(1)]; - tensor var_1166_0, tensor var_1166_1 = split(axis = var_1166_axis_0, split_sizes = var_1166_split_sizes_0, x = inputs_51)[name = tensor("op_1166")]; - tensor var_1168 = sigmoid(x = var_1166_1)[name = tensor("op_1168")]; - tensor inputs_53 = mul(x = var_1166_0, y = var_1168)[name = tensor("inputs_53")]; - tensor outputs_aug_13_pad_type_0 = const()[name = tensor("outputs_aug_13_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_13_pad_0 = const()[name = tensor("outputs_aug_13_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_13_groups_0 = const()[name = tensor("outputs_aug_13_groups_0"), val = tensor(256)]; - tensor outputs_aug_13_strides_0 = const()[name = tensor("outputs_aug_13_strides_0"), val = tensor([1])]; - tensor outputs_aug_13_dilations_0 = const()[name = tensor("outputs_aug_13_dilations_0"), val = tensor([1])]; - tensor outputs_aug_13 = conv(dilations = outputs_aug_13_dilations_0, groups = outputs_aug_13_groups_0, pad = outputs_aug_13_pad_0, pad_type = outputs_aug_13_pad_type_0, strides = outputs_aug_13_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_53)[name = tensor("outputs_aug_13")]; - tensor input_175_begin_0 = const()[name = tensor("input_175_begin_0"), val = tensor([0, 0, 0])]; - tensor input_175_end_0 = const()[name = tensor("input_175_end_0"), val = tensor([1, 256, 16])]; - tensor input_175_end_mask_0 = const()[name = tensor("input_175_end_mask_0"), val = tensor([true, true, false])]; - tensor input_175 = slice_by_index(begin = input_175_begin_0, end = input_175_end_0, end_mask = input_175_end_mask_0, x = outputs_aug_13)[name = tensor("input_175")]; - tensor inputs_55 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_175)[name = tensor("inputs_55")]; - tensor input_177 = silu(x = inputs_55)[name = tensor("input_177")]; - tensor input_179_pad_type_0 = const()[name = tensor("input_179_pad_type_0"), val = tensor("valid")]; - tensor input_179_strides_0 = const()[name = tensor("input_179_strides_0"), val = tensor([1])]; - tensor input_179_pad_0 = const()[name = tensor("input_179_pad_0"), val = tensor([0, 0])]; - tensor input_179_dilations_0 = const()[name = tensor("input_179_dilations_0"), val = tensor([1])]; - tensor input_179_groups_0 = const()[name = tensor("input_179_groups_0"), val = tensor(1)]; - tensor input_179 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_179_dilations_0, groups = input_179_groups_0, pad = input_179_pad_0, pad_type = input_179_pad_type_0, strides = input_179_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_177)[name = tensor("input_179")]; - tensor conv_out_13_perm_0 = const()[name = tensor("conv_out_13_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1199_begin_0 = const()[name = tensor("op_1199_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1199_end_0 = const()[name = tensor("op_1199_end_0"), val = tensor([1, 16, 256])]; - tensor var_1199_end_mask_0 = const()[name = tensor("op_1199_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_13 = transpose(perm = conv_out_13_perm_0, x = input_179)[name = tensor("transpose_33")]; - tensor var_1199 = slice_by_index(begin = var_1199_begin_0, end = var_1199_end_0, end_mask = var_1199_end_mask_0, x = conv_out_13)[name = tensor("op_1199")]; - tensor var_1202_begin_0 = const()[name = tensor("op_1202_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1202_end_0 = const()[name = tensor("op_1202_end_0"), val = tensor([1, 1, 256])]; - tensor var_1202_end_mask_0 = const()[name = tensor("op_1202_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1202 = slice_by_index(begin = var_1202_begin_0, end = var_1202_end_0, end_mask = var_1202_end_mask_0, x = x_27)[name = tensor("op_1202")]; - tensor var_1205_begin_0 = const()[name = tensor("op_1205_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1205_end_0 = const()[name = tensor("op_1205_end_0"), val = tensor([1, 16, 256])]; - tensor var_1205_end_mask_0 = const()[name = tensor("op_1205_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1205 = slice_by_index(begin = var_1205_begin_0, end = var_1205_end_0, end_mask = var_1205_end_mask_0, x = input_171)[name = tensor("op_1205")]; + tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_959)[name = tensor("x_expanded_13")]; + tensor var_973 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_973")]; + tensor var_974 = mul(x = window_19, y = var_973)[name = tensor("op_974")]; + tensor var_975 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_975")]; + tensor filled_window_13 = add(x = var_974, y = var_975)[name = tensor("filled_window_13")]; + tensor var_977 = greater_equal(x = filled_13, y = var_16)[name = tensor("op_977")]; + tensor var_978_dtype_0 = const()[name = tensor("op_978_dtype_0"), val = tensor("fp32")]; + tensor var_979 = const()[name = tensor("op_979"), val = tensor([1, 1, 1])]; + tensor var_978 = cast(dtype = var_978_dtype_0, x = var_977)[name = tensor("cast_130")]; + tensor use_roll_13 = reshape(shape = var_979, x = var_978)[name = tensor("use_roll_13")]; + tensor var_981 = sub(x = var_17, y = use_roll_13)[name = tensor("op_981")]; + tensor var_982 = mul(x = filled_window_13, y = var_981)[name = tensor("op_982")]; + tensor var_983 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_983")]; + tensor window_21 = add(x = var_982, y = var_983)[name = tensor("window_21")]; + tensor var_985 = const()[name = tensor("op_985"), val = tensor(0x1p+0)]; + tensor var_986 = add(x = filled_13, y = var_985)[name = tensor("op_986")]; + tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_986)[name = tensor("clip_13")]; + tensor var_989_begin_0 = const()[name = tensor("op_989_begin_0"), val = tensor([0, 1, 0])]; + tensor var_989_end_0 = const()[name = tensor("op_989_end_0"), val = tensor([1, 1, 256])]; + tensor var_989_end_mask_0 = const()[name = tensor("op_989_end_mask_0"), val = tensor([true, true, true])]; + tensor var_989 = slice_by_index(begin = var_989_begin_0, end = var_989_end_0, end_mask = var_989_end_mask_0, x = x_21)[name = tensor("op_989")]; + tensor var_992_begin_0 = const()[name = tensor("op_992_begin_0"), val = tensor([0, 1, 0])]; + tensor var_992_end_0 = const()[name = tensor("op_992_end_0"), val = tensor([1, 16, 256])]; + tensor var_992_end_mask_0 = const()[name = tensor("op_992_end_mask_0"), val = tensor([true, true, true])]; + tensor var_992 = slice_by_index(begin = var_992_begin_0, end = var_992_end_0, end_mask = var_992_end_mask_0, x = window_21)[name = tensor("op_992")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1205, var_1202))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_992, var_989))[name = tensor("rolled")]; tensor clip_14 = clip(alpha = var_21, beta = var_20, x = clip_13)[name = tensor("clip_14")]; - tensor var_1210 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_1210")]; - tensor var_1211_dtype_0 = const()[name = tensor("op_1211_dtype_0"), val = tensor("fp32")]; - tensor var_1212 = const()[name = tensor("op_1212"), val = tensor([1, 16, 1])]; - tensor var_1211 = cast(dtype = var_1211_dtype_0, x = var_1210)[name = tensor("cast_129")]; - tensor fill_mask = reshape(shape = var_1212, x = var_1211)[name = tensor("fill_mask")]; + tensor var_997 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_997")]; + tensor var_998_dtype_0 = const()[name = tensor("op_998_dtype_0"), val = tensor("fp32")]; + tensor var_999 = const()[name = tensor("op_999"), val = tensor([1, 16, 1])]; + tensor var_998 = cast(dtype = var_998_dtype_0, x = var_997)[name = tensor("cast_129")]; + tensor fill_mask = reshape(shape = var_999, x = var_998)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1202)[name = tensor("x_expanded")]; - tensor var_1216 = sub(x = var_17, y = fill_mask)[name = tensor("op_1216")]; - tensor var_1217 = mul(x = input_171, y = var_1216)[name = tensor("op_1217")]; - tensor var_1218 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1218")]; - tensor filled_window = add(x = var_1217, y = var_1218)[name = tensor("filled_window")]; - tensor var_1220 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_1220")]; - tensor var_1221_dtype_0 = const()[name = tensor("op_1221_dtype_0"), val = tensor("fp32")]; - tensor var_1222 = const()[name = tensor("op_1222"), val = tensor([1, 1, 1])]; - tensor var_1221 = cast(dtype = var_1221_dtype_0, x = var_1220)[name = tensor("cast_128")]; - tensor use_roll = reshape(shape = var_1222, x = var_1221)[name = tensor("use_roll")]; - tensor var_1224 = sub(x = var_17, y = use_roll)[name = tensor("op_1224")]; - tensor var_1225 = mul(x = filled_window, y = var_1224)[name = tensor("op_1225")]; - tensor var_1226 = mul(x = rolled, y = use_roll)[name = tensor("op_1226")]; - tensor input_181 = add(x = var_1225, y = var_1226)[name = tensor("input_181")]; - tensor var_1228 = const()[name = tensor("op_1228"), val = tensor(0x1p+0)]; - tensor var_1229 = add(x = clip_13, y = var_1228)[name = tensor("op_1229")]; - tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_1229)[name = tensor("clip_15")]; - tensor x_31_axes_0 = const()[name = tensor("x_31_axes_0"), val = tensor([-1])]; - tensor x_31 = layer_norm(axes = x_31_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_181)[name = tensor("x_31")]; - tensor input_183_perm_0 = const()[name = tensor("input_183_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_57_pad_type_0 = const()[name = tensor("inputs_57_pad_type_0"), val = tensor("valid")]; - tensor inputs_57_strides_0 = const()[name = tensor("inputs_57_strides_0"), val = tensor([1])]; - tensor inputs_57_pad_0 = const()[name = tensor("inputs_57_pad_0"), val = tensor([0, 0])]; - tensor inputs_57_dilations_0 = const()[name = tensor("inputs_57_dilations_0"), val = tensor([1])]; - tensor inputs_57_groups_0 = const()[name = tensor("inputs_57_groups_0"), val = tensor(1)]; - tensor input_183 = transpose(perm = input_183_perm_0, x = x_31)[name = tensor("transpose_32")]; - tensor inputs_57 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_57_dilations_0, groups = inputs_57_groups_0, pad = inputs_57_pad_0, pad_type = inputs_57_pad_type_0, strides = inputs_57_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_183)[name = tensor("inputs_57")]; - tensor var_1250_split_sizes_0 = const()[name = tensor("op_1250_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1250_axis_0 = const()[name = tensor("op_1250_axis_0"), val = tensor(1)]; - tensor var_1250_0, tensor var_1250_1 = split(axis = var_1250_axis_0, split_sizes = var_1250_split_sizes_0, x = inputs_57)[name = tensor("op_1250")]; - tensor var_1252 = sigmoid(x = var_1250_1)[name = tensor("op_1252")]; - tensor inputs_59 = mul(x = var_1250_0, y = var_1252)[name = tensor("inputs_59")]; + tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_989)[name = tensor("x_expanded")]; + tensor var_1003 = sub(x = var_17, y = fill_mask)[name = tensor("op_1003")]; + tensor var_1004 = mul(x = window_21, y = var_1003)[name = tensor("op_1004")]; + tensor var_1005 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1005")]; + tensor filled_window = add(x = var_1004, y = var_1005)[name = tensor("filled_window")]; + tensor var_1007 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_1007")]; + tensor var_1008_dtype_0 = const()[name = tensor("op_1008_dtype_0"), val = tensor("fp32")]; + tensor var_1009 = const()[name = tensor("op_1009"), val = tensor([1, 1, 1])]; + tensor var_1008 = cast(dtype = var_1008_dtype_0, x = var_1007)[name = tensor("cast_128")]; + tensor use_roll = reshape(shape = var_1009, x = var_1008)[name = tensor("use_roll")]; + tensor var_1011 = sub(x = var_17, y = use_roll)[name = tensor("op_1011")]; + tensor var_1012 = mul(x = filled_window, y = var_1011)[name = tensor("op_1012")]; + tensor var_1013 = mul(x = rolled, y = use_roll)[name = tensor("op_1013")]; + tensor window = add(x = var_1012, y = var_1013)[name = tensor("window")]; + tensor var_1015 = const()[name = tensor("op_1015"), val = tensor(0x1p+0)]; + tensor var_1016 = add(x = clip_13, y = var_1015)[name = tensor("op_1016")]; + tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_1016)[name = tensor("clip_15")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_30, interleave = input_141_interleave_0, values = (window_21, window))[name = tensor("input_141")]; + tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; + tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; + tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_33_pad_type_0 = const()[name = tensor("inputs_33_pad_type_0"), val = tensor("valid")]; + tensor inputs_33_strides_0 = const()[name = tensor("inputs_33_strides_0"), val = tensor([1])]; + tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; + tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; + tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; + tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; + tensor var_1039_split_sizes_0 = const()[name = tensor("op_1039_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1039_axis_0 = const()[name = tensor("op_1039_axis_0"), val = tensor(1)]; + tensor var_1039_0, tensor var_1039_1 = split(axis = var_1039_axis_0, split_sizes = var_1039_split_sizes_0, x = inputs_33)[name = tensor("op_1039")]; + tensor var_1041 = sigmoid(x = var_1039_1)[name = tensor("op_1041")]; + tensor inputs_35 = mul(x = var_1039_0, y = var_1041)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; tensor outputs_aug_strides_0 = const()[name = tensor("outputs_aug_strides_0"), val = tensor([1])]; tensor outputs_aug_dilations_0 = const()[name = tensor("outputs_aug_dilations_0"), val = tensor([1])]; - tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_59)[name = tensor("outputs_aug")]; - tensor input_185_begin_0 = const()[name = tensor("input_185_begin_0"), val = tensor([0, 0, 0])]; - tensor input_185_end_0 = const()[name = tensor("input_185_end_0"), val = tensor([1, 256, 16])]; - tensor input_185_end_mask_0 = const()[name = tensor("input_185_end_mask_0"), val = tensor([true, true, false])]; - tensor input_185 = slice_by_index(begin = input_185_begin_0, end = input_185_end_0, end_mask = input_185_end_mask_0, x = outputs_aug)[name = tensor("input_185")]; - tensor inputs_61 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_185)[name = tensor("inputs_61")]; - tensor input_187 = silu(x = inputs_61)[name = tensor("input_187")]; - tensor input_189_pad_type_0 = const()[name = tensor("input_189_pad_type_0"), val = tensor("valid")]; - tensor input_189_strides_0 = const()[name = tensor("input_189_strides_0"), val = tensor([1])]; - tensor input_189_pad_0 = const()[name = tensor("input_189_pad_0"), val = tensor([0, 0])]; - tensor input_189_dilations_0 = const()[name = tensor("input_189_dilations_0"), val = tensor([1])]; - tensor input_189_groups_0 = const()[name = tensor("input_189_groups_0"), val = tensor(1)]; - tensor input_189 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_189_dilations_0, groups = input_189_groups_0, pad = input_189_pad_0, pad_type = input_189_pad_type_0, strides = input_189_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_187)[name = tensor("input_189")]; - tensor conv_out_15_perm_0 = const()[name = tensor("conv_out_15_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1283_begin_0 = const()[name = tensor("op_1283_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1283_end_0 = const()[name = tensor("op_1283_end_0"), val = tensor([1, 16, 256])]; - tensor var_1283_end_mask_0 = const()[name = tensor("op_1283_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_15 = transpose(perm = conv_out_15_perm_0, x = input_189)[name = tensor("transpose_31")]; - tensor var_1283 = slice_by_index(begin = var_1283_begin_0, end = var_1283_end_0, end_mask = var_1283_end_mask_0, x = conv_out_15)[name = tensor("op_1283")]; - tensor var_1286_interleave_0 = const()[name = tensor("op_1286_interleave_0"), val = tensor(false)]; - tensor var_1286 = concat(axis = var_33, interleave = var_1286_interleave_0, values = (var_1199, var_1283))[name = tensor("op_1286")]; - tensor input_191 = add(x = x_27, y = var_1286)[name = tensor("input_191")]; - tensor input_193_axes_0 = const()[name = tensor("input_193_axes_0"), val = tensor([-1])]; - tensor input_193 = layer_norm(axes = input_193_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_191)[name = tensor("input_193")]; - tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_193)[name = tensor("linear_35")]; - tensor input_195 = silu(x = inputs)[name = tensor("input_195")]; - tensor input_199 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_195)[name = tensor("linear_36")]; - tensor var_1309 = const()[name = tensor("op_1309"), val = tensor(0x1p-1)]; - tensor var_1310 = mul(x = input_199, y = var_1309)[name = tensor("op_1310")]; - tensor input_201 = add(x = var_1310, y = input_191)[name = tensor("input_201")]; - tensor x_33_axes_0 = const()[name = tensor("x_33_axes_0"), val = tensor([-1])]; - tensor x_33 = layer_norm(axes = x_33_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_201)[name = tensor("x_33")]; + tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_35)[name = tensor("outputs_aug")]; + tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; + tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([2, 256, 16])]; + tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; + tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug)[name = tensor("input_145")]; + tensor inputs_37 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_145)[name = tensor("inputs_37")]; + tensor input_147 = silu(x = inputs_37)[name = tensor("input_147")]; + tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; + tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; + tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; + tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; + tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; + tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; + tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1072_begin_0 = const()[name = tensor("op_1072_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1072_end_0 = const()[name = tensor("op_1072_end_0"), val = tensor([2, 16, 256])]; + tensor var_1072_end_mask_0 = const()[name = tensor("op_1072_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_1072 = slice_by_index(begin = var_1072_begin_0, end = var_1072_end_0, end_mask = var_1072_end_mask_0, x = conv_out_7)[name = tensor("op_1072")]; + tensor var_1074_perm_0 = const()[name = tensor("op_1074_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1074 = transpose(perm = var_1074_perm_0, x = var_1072)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_1074)[name = tensor("input_151")]; + tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; + tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; + tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; + tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; + tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; + tensor var_1097 = const()[name = tensor("op_1097"), val = tensor(0x1p-1)]; + tensor var_1098 = mul(x = input_159, y = var_1097)[name = tensor("op_1098")]; + tensor input_161 = add(x = var_1098, y = input_151)[name = tensor("input_161")]; + tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; + tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; tensor cat_interleave_0 = const()[name = tensor("cat_interleave_0"), val = tensor(false)]; - tensor x_ct = transpose(perm = x_ct_perm_0, x = x_33)[name = tensor("transpose_30")]; + tensor x_ct = transpose(perm = x_ct_perm_0, x = x_25)[name = tensor("transpose_30")]; tensor cat = concat(axis = var_27, interleave = cat_interleave_0, values = (cnn_window, x_ct))[name = tensor("cat")]; tensor conv_out_pad_type_0 = const()[name = tensor("conv_out_pad_type_0"), val = tensor("valid")]; tensor conv_out_strides_0 = const()[name = tensor("conv_out_strides_0"), val = tensor([1])]; @@ -1221,44 +1074,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_1328_begin_0 = const()[name = tensor("op_1328_begin_0"), val = tensor([0, 0, 2])]; - tensor var_1328_end_0 = const()[name = tensor("op_1328_end_0"), val = tensor([1, 256, 20])]; - tensor var_1328_end_mask_0 = const()[name = tensor("op_1328_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_1328_begin_0, end = var_1328_end_0, end_mask = var_1328_end_mask_0, x = cat)[name = tensor("op_1328")]; - tensor input_203_perm_0 = const()[name = tensor("input_203_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1330 = const()[name = tensor("op_1330"), val = tensor([-1])]; - tensor input_203 = transpose(perm = input_203_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_1331 = reduce_l2_norm(axes = var_1330, keep_dims = var_36, x = input_203)[name = tensor("op_1331")]; - tensor const_16 = const()[name = tensor("const_16"), val = tensor(0x1.fffffep+127)]; - tensor clip_16 = clip(alpha = var_12, beta = const_16, x = var_1331)[name = tensor("clip_16")]; - tensor emb = real_div(x = input_203, y = clip_16)[name = tensor("emb")]; - tensor var_1335_axis_0 = const()[name = tensor("op_1335_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_1335_axis_0, values = (var_214, var_510, var_806, nkv_1))[name = tensor("op_1335")]; - tensor var_1337_axis_0 = const()[name = tensor("op_1337_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_1337_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1337")]; - tensor var_1339_axis_0 = const()[name = tensor("op_1339_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_1339_axis_0, values = (input_31, input_81, input_131, input_181))[name = tensor("op_1339")]; - tensor var_1341_axis_0 = const()[name = tensor("op_1341_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_1341_axis_0, values = (clip_3, clip_7, clip_11, clip_15))[name = tensor("op_1341")]; - tensor var_1351 = const()[name = tensor("op_1351"), val = tensor(0x1.5798eep-27)]; - tensor var_1357 = const()[name = tensor("op_1357"), val = tensor(0x1.4f8b58p-17)]; - tensor var_1359 = const()[name = tensor("op_1359"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_1360 = const()[name = tensor("op_1360"), val = tensor(true)]; - tensor var_1362 = const()[name = tensor("op_1362"), val = tensor(0x1p+0)]; - tensor var_1366 = const()[name = tensor("op_1366"), val = tensor(-1)]; - tensor var_1371 = const()[name = tensor("op_1371"), val = tensor(0)]; + tensor var_1116_begin_0 = const()[name = tensor("op_1116_begin_0"), val = tensor([0, 0, 2])]; + tensor var_1116_end_0 = const()[name = tensor("op_1116_end_0"), val = tensor([1, 256, 20])]; + tensor var_1116_end_mask_0 = const()[name = tensor("op_1116_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_1116_begin_0, end = var_1116_end_0, end_mask = var_1116_end_mask_0, x = cat)[name = tensor("op_1116")]; + tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1118 = const()[name = tensor("op_1118"), val = tensor([-1])]; + tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; + tensor var_1119 = reduce_l2_norm(axes = var_1118, keep_dims = var_36, x = input_163)[name = tensor("op_1119")]; + tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; + tensor clip_16 = clip(alpha = var_12, beta = const_12, x = var_1119)[name = tensor("clip_16")]; + tensor emb = real_div(x = input_163, y = clip_16)[name = tensor("emb")]; + tensor var_1123_axis_0 = const()[name = tensor("op_1123_axis_0"), val = tensor(0)]; + tensor enc_kv_new = stack(axis = var_1123_axis_0, values = (var_214, var_457, var_700, nkv_1))[name = tensor("op_1123")]; + tensor var_1125_axis_0 = const()[name = tensor("op_1125_axis_0"), val = tensor(0)]; + tensor enc_scale_new = stack(axis = var_1125_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1125")]; + tensor var_1127_axis_0 = const()[name = tensor("op_1127_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1127_axis_0, values = (window_5, window_11, window_17, window))[name = tensor("op_1127")]; + tensor var_1129_axis_0 = const()[name = tensor("op_1129_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1129_axis_0, values = (clip_3, clip_7, clip_11, clip_15))[name = tensor("op_1129")]; + tensor var_1139 = const()[name = tensor("op_1139"), val = tensor(0x1.5798eep-27)]; + tensor var_1144 = const()[name = tensor("op_1144"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1146 = const()[name = tensor("op_1146"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1147 = const()[name = tensor("op_1147"), val = tensor(true)]; + tensor var_1149 = const()[name = tensor("op_1149"), val = tensor(0x1p+0)]; + tensor var_1153 = const()[name = tensor("op_1153"), val = tensor(-1)]; + tensor var_1159 = const()[name = tensor("op_1159"), val = tensor(0)]; tensor pos = const()[name = tensor("pos"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395456)))]; - tensor var_1433_axes_0 = const()[name = tensor("op_1433_axes_0"), val = tensor([2])]; - tensor var_1433 = expand_dims(axes = var_1433_axes_0, x = emb)[name = tensor("op_1433")]; - tensor var_1434 = const()[name = tensor("op_1434"), val = tensor([1, 1, 12, 1])]; - tensor emb_exp = tile(reps = var_1434, x = var_1433)[name = tensor("emb_exp")]; - tensor input_205_interleave_0 = const()[name = tensor("input_205_interleave_0"), val = tensor(false)]; - tensor input_205 = concat(axis = var_1366, interleave = input_205_interleave_0, values = (emb_exp, pos))[name = tensor("input_205")]; - tensor x_35 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_205)[name = tensor("linear_37")]; - tensor var_1441_perm_0 = const()[name = tensor("op_1441_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1445 = const()[name = tensor("op_1445"), val = tensor([12, 2, 256])]; - tensor var_1441 = transpose(perm = var_1441_perm_0, x = x_35)[name = tensor("transpose_28")]; - tensor x_37 = reshape(shape = var_1445, x = var_1441)[name = tensor("x_37")]; + tensor var_1221_axes_0 = const()[name = tensor("op_1221_axes_0"), val = tensor([2])]; + tensor var_1221 = expand_dims(axes = var_1221_axes_0, x = emb)[name = tensor("op_1221")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 12, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1221)[name = tensor("emb_exp")]; + tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; + tensor input_165 = concat(axis = var_1153, interleave = input_165_interleave_0, values = (emb_exp, pos))[name = tensor("input_165")]; + tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; + tensor var_1229_perm_0 = const()[name = tensor("op_1229_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1233 = const()[name = tensor("op_1233"), val = tensor([12, 2, 256])]; + tensor var_1229 = transpose(perm = var_1229_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1233, x = var_1229)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 12, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1269,132 +1122,132 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_1453 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_37)[name = tensor("linear_38")]; - tensor var_1454 = const()[name = tensor("op_1454"), val = tensor([12, 2, 4, 64])]; - tensor var_1455 = reshape(shape = var_1454, x = var_1453)[name = tensor("op_1455")]; + tensor var_1241 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1242 = const()[name = tensor("op_1242"), val = tensor([12, 2, 4, 64])]; + tensor var_1243 = reshape(shape = var_1242, x = var_1241)[name = tensor("op_1243")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1459 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_37)[name = tensor("linear_39")]; - tensor var_1460 = const()[name = tensor("op_1460"), val = tensor(0x1p-3)]; - tensor var_1461 = mul(x = var_1459, y = var_1460)[name = tensor("op_1461")]; - tensor var_1462 = const()[name = tensor("op_1462"), val = tensor([12, 2, 4, 64])]; - tensor var_1463 = reshape(shape = var_1462, x = var_1461)[name = tensor("op_1463")]; + tensor var_1247 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1248 = const()[name = tensor("op_1248"), val = tensor(0x1p-3)]; + tensor var_1249 = mul(x = var_1247, y = var_1248)[name = tensor("op_1249")]; + tensor var_1250 = const()[name = tensor("op_1250"), val = tensor([12, 2, 4, 64])]; + tensor var_1251 = reshape(shape = var_1250, x = var_1249)[name = tensor("op_1251")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1467 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_37)[name = tensor("linear_40")]; - tensor var_1468 = const()[name = tensor("op_1468"), val = tensor([12, 2, 4, 64])]; - tensor var_1469 = reshape(shape = var_1468, x = var_1467)[name = tensor("op_1469")]; + tensor var_1255 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1256 = const()[name = tensor("op_1256"), val = tensor([12, 2, 4, 64])]; + tensor var_1257 = reshape(shape = var_1256, x = var_1255)[name = tensor("op_1257")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_209 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_37)[name = tensor("linear_41")]; + tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_1371, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1159, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; - tensor const_24 = const()[name = tensor("const_24"), val = tensor(0x1.fffffep+127)]; - tensor clip_17 = clip(alpha = var_1362, beta = const_24, x = s_t_9)[name = tensor("clip_17")]; + tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; + tensor clip_17 = clip(alpha = var_1149, beta = const_20, x = s_t_9)[name = tensor("clip_17")]; tensor sqrt_s_t_9 = sqrt(x = clip_17)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_1463)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_1455)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1251)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1243)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_1481 = const()[name = tensor("op_1481"), val = tensor([1, 2])]; - tensor var_1482 = reshape(shape = var_1481, x = valid_mask)[name = tensor("op_1482")]; - tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1482)[name = tensor("causal_with_valid_1")]; - tensor var_1484 = const()[name = tensor("op_1484"), val = tensor([2, 1])]; - tensor var_1485 = reshape(shape = var_1484, x = sqrt_s_t_9)[name = tensor("op_1485")]; - tensor M_9 = real_div(x = causal_with_valid_1, y = var_1485)[name = tensor("M_9")]; - tensor var_1487 = mul(x = qk_9, y = M_9)[name = tensor("op_1487")]; + tensor var_1269 = const()[name = tensor("op_1269"), val = tensor([1, 2])]; + tensor var_1270 = reshape(shape = var_1269, x = valid_mask)[name = tensor("op_1270")]; + tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1270)[name = tensor("causal_with_valid_1")]; + tensor var_1272 = const()[name = tensor("op_1272"), val = tensor([2, 1])]; + tensor var_1273 = reshape(shape = var_1272, x = sqrt_s_t_9)[name = tensor("op_1273")]; + tensor M_9 = real_div(x = causal_with_valid_1, y = var_1273)[name = tensor("M_9")]; + tensor var_1275 = mul(x = qk_9, y = M_9)[name = tensor("op_1275")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_1469)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1487, y = v_9)[name = tensor("inner_9")]; - tensor var_1489_transpose_x_0 = const()[name = tensor("op_1489_transpose_x_0"), val = tensor(false)]; - tensor var_1489_transpose_y_0 = const()[name = tensor("op_1489_transpose_y_0"), val = tensor(false)]; - tensor var_1489 = matmul(transpose_x = var_1489_transpose_x_0, transpose_y = var_1489_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1489")]; - tensor var_1490 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1490")]; - tensor var_1491 = const()[name = tensor("op_1491"), val = tensor([1, 1, 2, 1])]; - tensor var_1492 = reshape(shape = var_1491, x = var_1490)[name = tensor("op_1492")]; - tensor cross_9 = mul(x = var_1489, y = var_1492)[name = tensor("cross_9")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1257)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1275, y = v_9)[name = tensor("inner_9")]; + tensor var_1277_transpose_x_0 = const()[name = tensor("op_1277_transpose_x_0"), val = tensor(false)]; + tensor var_1277_transpose_y_0 = const()[name = tensor("op_1277_transpose_y_0"), val = tensor(false)]; + tensor var_1277 = matmul(transpose_x = var_1277_transpose_x_0, transpose_y = var_1277_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1277")]; + tensor var_1278 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1278")]; + tensor var_1279 = const()[name = tensor("op_1279"), val = tensor([1, 1, 2, 1])]; + tensor var_1280 = reshape(shape = var_1279, x = var_1278)[name = tensor("op_1280")]; + tensor cross_9 = mul(x = var_1277, y = var_1280)[name = tensor("cross_9")]; tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; - tensor var_1495 = const()[name = tensor("op_1495"), val = tensor([1, 1, 2, 1])]; - tensor var_1496 = reshape(shape = var_1495, x = valid_mask)[name = tensor("op_1496")]; - tensor v_masked_1 = mul(x = v_9, y = var_1496)[name = tensor("v_masked_1")]; - tensor var_1498 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1498")]; - tensor var_1500_transpose_x_1 = const()[name = tensor("op_1500_transpose_x_1"), val = tensor(true)]; - tensor var_1500_transpose_y_1 = const()[name = tensor("op_1500_transpose_y_1"), val = tensor(false)]; - tensor var_1500 = matmul(transpose_x = var_1500_transpose_x_1, transpose_y = var_1500_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1500")]; - tensor new_kv_unnorm_9 = add(x = var_1498, y = var_1500)[name = tensor("new_kv_unnorm_9")]; - tensor var_1502_keep_dims_0 = const()[name = tensor("op_1502_keep_dims_0"), val = tensor(false)]; - tensor var_1502 = reduce_sum(keep_dims = var_1502_keep_dims_0, x = valid_mask)[name = tensor("op_1502")]; - tensor var_1503 = const()[name = tensor("op_1503"), val = tensor([1])]; - tensor var_1504 = reshape(shape = var_1503, x = var_1502)[name = tensor("op_1504")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_1504)[name = tensor("new_scale_9")]; - tensor const_25 = const()[name = tensor("const_25"), val = tensor(0x1.fffffep+127)]; - tensor clip_18 = clip(alpha = var_1362, beta = const_25, x = new_scale_9)[name = tensor("clip_18")]; + tensor var_1283 = const()[name = tensor("op_1283"), val = tensor([1, 1, 2, 1])]; + tensor var_1284 = reshape(shape = var_1283, x = valid_mask)[name = tensor("op_1284")]; + tensor v_masked_1 = mul(x = v_9, y = var_1284)[name = tensor("v_masked_1")]; + tensor var_1286 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1286")]; + tensor var_1288_transpose_x_1 = const()[name = tensor("op_1288_transpose_x_1"), val = tensor(true)]; + tensor var_1288_transpose_y_1 = const()[name = tensor("op_1288_transpose_y_1"), val = tensor(false)]; + tensor var_1288 = matmul(transpose_x = var_1288_transpose_x_1, transpose_y = var_1288_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1288")]; + tensor new_kv_unnorm_9 = add(x = var_1286, y = var_1288)[name = tensor("new_kv_unnorm_9")]; + tensor var_1290_keep_dims_0 = const()[name = tensor("op_1290_keep_dims_0"), val = tensor(false)]; + tensor var_1290 = reduce_sum(keep_dims = var_1290_keep_dims_0, x = valid_mask)[name = tensor("op_1290")]; + tensor var_1291 = const()[name = tensor("op_1291"), val = tensor([1])]; + tensor var_1292 = reshape(shape = var_1291, x = var_1290)[name = tensor("op_1292")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1292)[name = tensor("new_scale_9")]; + tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; + tensor clip_18 = clip(alpha = var_1149, beta = const_21, x = new_scale_9)[name = tensor("clip_18")]; tensor sqrt_new_scale_1 = sqrt(x = clip_18)[name = tensor("sqrt_new_scale_1")]; - tensor var_1508 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1508")]; - tensor var_1509_perm_0 = const()[name = tensor("op_1509_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1296 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1296")]; + tensor var_1297_perm_0 = const()[name = tensor("op_1297_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_1509 = transpose(perm = var_1509_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1359, x = var_1509)[name = tensor("out_27")]; - tensor var_1513 = const()[name = tensor("op_1513"), val = tensor([12, 2, 256])]; - tensor out_29 = reshape(shape = var_1513, x = out_27)[name = tensor("out_29")]; - tensor var_1515 = silu(x = input_209)[name = tensor("op_1515")]; - tensor input_211 = mul(x = var_1515, y = out_29)[name = tensor("input_211")]; - tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_211)[name = tensor("linear_42")]; - tensor input_213 = add(x = x_37, y = ret_out_9)[name = tensor("input_213")]; + tensor var_1297 = transpose(perm = var_1297_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1146, x = var_1297)[name = tensor("out_27")]; + tensor var_1301 = const()[name = tensor("op_1301"), val = tensor([12, 2, 256])]; + tensor out_29 = reshape(shape = var_1301, x = out_27)[name = tensor("out_29")]; + tensor var_1303 = silu(x = input_169)[name = tensor("op_1303")]; + tensor input_171 = mul(x = var_1303, y = out_29)[name = tensor("input_171")]; + tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; + tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1357, gamma = decoder_norm11_0_weight, x = input_213)[name = tensor("xt_1")]; - tensor var_1525 = const()[name = tensor("op_1525"), val = tensor([1, 12, 2, 256])]; - tensor var_1526 = reshape(shape = var_1525, x = xt_1)[name = tensor("op_1526")]; - tensor var_1527_perm_0 = const()[name = tensor("op_1527_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1530 = const()[name = tensor("op_1530"), val = tensor([2, 12, 256])]; - tensor var_1527 = transpose(perm = var_1527_perm_0, x = var_1526)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_1530, x = var_1527)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1144, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1313 = const()[name = tensor("op_1313"), val = tensor([1, 12, 2, 256])]; + tensor var_1314 = reshape(shape = var_1313, x = xt_1)[name = tensor("op_1314")]; + tensor var_1315_perm_0 = const()[name = tensor("op_1315_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1318 = const()[name = tensor("op_1318"), val = tensor([2, 12, 256])]; + tensor var_1315 = transpose(perm = var_1315_perm_0, x = var_1314)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1318, x = var_1315)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_1553 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_8 = const()[name = tensor("concat_8"), val = tensor([12, 2, 3, 256])]; - tensor var_1555 = reshape(shape = concat_8, x = var_1553)[name = tensor("op_1555")]; - tensor var_1556_axes_0 = const()[name = tensor("op_1556_axes_0"), val = tensor([0])]; - tensor var_1556 = expand_dims(axes = var_1556_axes_0, x = var_1555)[name = tensor("op_1556")]; - tensor var_1557_perm_0 = const()[name = tensor("op_1557_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1558_axes_0 = const()[name = tensor("op_1558_axes_0"), val = tensor([-2])]; - tensor var_1557 = transpose(perm = var_1557_perm_0, x = var_1556)[name = tensor("transpose_21")]; - tensor var_1558 = squeeze(axes = var_1558_axes_0, x = var_1557)[name = tensor("op_1558")]; + tensor var_1341 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_9 = const()[name = tensor("concat_9"), val = tensor([12, 2, 3, 256])]; + tensor var_1343 = reshape(shape = concat_9, x = var_1341)[name = tensor("op_1343")]; + tensor var_1344_axes_0 = const()[name = tensor("op_1344_axes_0"), val = tensor([0])]; + tensor var_1344 = expand_dims(axes = var_1344_axes_0, x = var_1343)[name = tensor("op_1344")]; + tensor var_1345_perm_0 = const()[name = tensor("op_1345_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1346_axes_0 = const()[name = tensor("op_1346_axes_0"), val = tensor([-2])]; + tensor var_1345 = transpose(perm = var_1345_perm_0, x = var_1344)[name = tensor("transpose_21")]; + tensor var_1346 = squeeze(axes = var_1346_axes_0, x = var_1345)[name = tensor("op_1346")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 12, 2, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1558)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1346)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 12, 2, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1558)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1346)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 12, 2, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1558)[name = tensor("v_11")]; - tensor var_1566 = const()[name = tensor("op_1566"), val = tensor([12, 8, 64])]; - tensor var_1567 = reshape(shape = var_1566, x = q_11)[name = tensor("op_1567")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1346)[name = tensor("v_11")]; + tensor var_1354 = const()[name = tensor("op_1354"), val = tensor([12, 8, 64])]; + tensor var_1355 = reshape(shape = var_1354, x = q_11)[name = tensor("op_1355")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1573 = const()[name = tensor("op_1573"), val = tensor([12, 8, 64])]; - tensor var_1574 = reshape(shape = var_1573, x = k_11)[name = tensor("op_1574")]; + tensor var_1361 = const()[name = tensor("op_1361"), val = tensor([12, 8, 64])]; + tensor var_1362 = reshape(shape = var_1361, x = k_11)[name = tensor("op_1362")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1580 = const()[name = tensor("op_1580"), val = tensor([12, 8, 64])]; - tensor var_1581 = reshape(shape = var_1580, x = v_11)[name = tensor("op_1581")]; + tensor var_1368 = const()[name = tensor("op_1368"), val = tensor([12, 8, 64])]; + tensor var_1369 = reshape(shape = var_1368, x = v_11)[name = tensor("op_1369")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1584 = const()[name = tensor("op_1584"), val = tensor([2, 4, 12, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_1567)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_1584, x = q_13)[name = tensor("q_15")]; - tensor var_1586 = const()[name = tensor("op_1586"), val = tensor([2, 4, 12, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_1574)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_1586, x = k_13)[name = tensor("k_15")]; - tensor var_1588 = const()[name = tensor("op_1588"), val = tensor([2, 4, 12, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_1581)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_1588, x = v_13)[name = tensor("v_15")]; + tensor var_1372 = const()[name = tensor("op_1372"), val = tensor([2, 4, 12, 64])]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1355)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1372, x = q_13)[name = tensor("q_15")]; + tensor var_1374 = const()[name = tensor("op_1374"), val = tensor([2, 4, 12, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1362)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1374, x = k_13)[name = tensor("k_15")]; + tensor var_1376 = const()[name = tensor("op_1376"), val = tensor([2, 4, 12, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1369)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1376, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -1405,30 +1258,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_1591 = const()[name = tensor("op_1591"), val = tensor([2, 0, 1, 3])]; - tensor var_1596 = const()[name = tensor("op_1596"), val = tensor([24, 256])]; - tensor var_1592 = transpose(perm = var_1591, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_1596, x = var_1592)[name = tensor("attn_output_3")]; + tensor var_1379 = const()[name = tensor("op_1379"), val = tensor([2, 0, 1, 3])]; + tensor var_1384 = const()[name = tensor("op_1384"), val = tensor([24, 256])]; + tensor var_1380 = transpose(perm = var_1379, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1384, x = var_1380)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_1600 = const()[name = tensor("op_1600"), val = tensor([12, 2, 256])]; - tensor attn_output_7 = reshape(shape = var_1600, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1388 = const()[name = tensor("op_1388"), val = tensor([12, 2, 256])]; + tensor attn_output_7 = reshape(shape = var_1388, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; - tensor input_215 = add(x = query_1, y = sa2_out_1)[name = tensor("input_215")]; - tensor input_217_axes_0 = const()[name = tensor("input_217_axes_0"), val = tensor([-1])]; - tensor input_217 = layer_norm(axes = input_217_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1357, gamma = decoder_norm21_0_weight, x = input_215)[name = tensor("input_217")]; - tensor input_219 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_217)[name = tensor("linear_45")]; - tensor input_221 = relu(x = input_219)[name = tensor("input_221")]; - tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_221)[name = tensor("linear_46")]; - tensor input_223 = add(x = input_217, y = ffn_out_1)[name = tensor("input_223")]; + tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; + tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1144, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; + tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; + tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; + tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1357, gamma = decoder_norm22_0_weight, x = input_223)[name = tensor("xt_3")]; - tensor var_1620 = const()[name = tensor("op_1620"), val = tensor([1, 2, 12, 256])]; - tensor x_39 = reshape(shape = var_1620, x = xt_3)[name = tensor("x_39")]; - tensor var_1622_perm_0 = const()[name = tensor("op_1622_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1626 = const()[name = tensor("op_1626"), val = tensor([12, 2, 256])]; - tensor var_1622 = transpose(perm = var_1622_perm_0, x = x_39)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_1626, x = var_1622)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1144, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1408 = const()[name = tensor("op_1408"), val = tensor([1, 2, 12, 256])]; + tensor x_31 = reshape(shape = var_1408, x = xt_3)[name = tensor("x_31")]; + tensor var_1410_perm_0 = const()[name = tensor("op_1410_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1414 = const()[name = tensor("op_1414"), val = tensor([12, 2, 256])]; + tensor var_1410 = transpose(perm = var_1410_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1414, x = var_1410)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 12, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1439,120 +1292,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_1634 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_1635 = const()[name = tensor("op_1635"), val = tensor([12, 2, 4, 64])]; - tensor var_1636 = reshape(shape = var_1635, x = var_1634)[name = tensor("op_1636")]; + tensor var_1422 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1423 = const()[name = tensor("op_1423"), val = tensor([12, 2, 4, 64])]; + tensor var_1424 = reshape(shape = var_1423, x = var_1422)[name = tensor("op_1424")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1640 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_1641 = const()[name = tensor("op_1641"), val = tensor(0x1p-3)]; - tensor var_1642 = mul(x = var_1640, y = var_1641)[name = tensor("op_1642")]; - tensor var_1643 = const()[name = tensor("op_1643"), val = tensor([12, 2, 4, 64])]; - tensor var_1644 = reshape(shape = var_1643, x = var_1642)[name = tensor("op_1644")]; + tensor var_1428 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1429 = const()[name = tensor("op_1429"), val = tensor(0x1p-3)]; + tensor var_1430 = mul(x = var_1428, y = var_1429)[name = tensor("op_1430")]; + tensor var_1431 = const()[name = tensor("op_1431"), val = tensor([12, 2, 4, 64])]; + tensor var_1432 = reshape(shape = var_1431, x = var_1430)[name = tensor("op_1432")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1648 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_1649 = const()[name = tensor("op_1649"), val = tensor([12, 2, 4, 64])]; - tensor var_1650 = reshape(shape = var_1649, x = var_1648)[name = tensor("op_1650")]; + tensor var_1436 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1437 = const()[name = tensor("op_1437"), val = tensor([12, 2, 4, 64])]; + tensor var_1438 = reshape(shape = var_1437, x = var_1436)[name = tensor("op_1438")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_227 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; + tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; - tensor const_36 = const()[name = tensor("const_36"), val = tensor(0x1.fffffep+127)]; - tensor clip_19 = clip(alpha = var_1362, beta = const_36, x = s_t)[name = tensor("clip_19")]; + tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; + tensor clip_19 = clip(alpha = var_1149, beta = const_32, x = s_t)[name = tensor("clip_19")]; tensor sqrt_s_t = sqrt(x = clip_19)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_1644)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_1636)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1432)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1424)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_1665 = const()[name = tensor("op_1665"), val = tensor([2, 1])]; - tensor var_1666 = reshape(shape = var_1665, x = sqrt_s_t)[name = tensor("op_1666")]; - tensor M = real_div(x = causal_with_valid_1, y = var_1666)[name = tensor("M")]; - tensor var_1668 = mul(x = qk, y = M)[name = tensor("op_1668")]; + tensor var_1453 = const()[name = tensor("op_1453"), val = tensor([2, 1])]; + tensor var_1454 = reshape(shape = var_1453, x = sqrt_s_t)[name = tensor("op_1454")]; + tensor M = real_div(x = causal_with_valid_1, y = var_1454)[name = tensor("M")]; + tensor var_1456 = mul(x = qk, y = M)[name = tensor("op_1456")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_1650)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1668, y = v_17)[name = tensor("inner")]; - tensor var_1670_transpose_x_0 = const()[name = tensor("op_1670_transpose_x_0"), val = tensor(false)]; - tensor var_1670_transpose_y_0 = const()[name = tensor("op_1670_transpose_y_0"), val = tensor(false)]; - tensor var_1670 = matmul(transpose_x = var_1670_transpose_x_0, transpose_y = var_1670_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1670")]; - tensor var_1671 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1671")]; - tensor var_1672 = const()[name = tensor("op_1672"), val = tensor([1, 1, 2, 1])]; - tensor var_1673 = reshape(shape = var_1672, x = var_1671)[name = tensor("op_1673")]; - tensor cross = mul(x = var_1670, y = var_1673)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1438)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1456, y = v_17)[name = tensor("inner")]; + tensor var_1458_transpose_x_0 = const()[name = tensor("op_1458_transpose_x_0"), val = tensor(false)]; + tensor var_1458_transpose_y_0 = const()[name = tensor("op_1458_transpose_y_0"), val = tensor(false)]; + tensor var_1458 = matmul(transpose_x = var_1458_transpose_x_0, transpose_y = var_1458_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1458")]; + tensor var_1459 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1459")]; + tensor var_1460 = const()[name = tensor("op_1460"), val = tensor([1, 1, 2, 1])]; + tensor var_1461 = reshape(shape = var_1460, x = var_1459)[name = tensor("op_1461")]; + tensor cross = mul(x = var_1458, y = var_1461)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_1496)[name = tensor("v_masked")]; - tensor var_1679 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1679")]; - tensor var_1681_transpose_x_1 = const()[name = tensor("op_1681_transpose_x_1"), val = tensor(true)]; - tensor var_1681_transpose_y_1 = const()[name = tensor("op_1681_transpose_y_1"), val = tensor(false)]; - tensor var_1681 = matmul(transpose_x = var_1681_transpose_x_1, transpose_y = var_1681_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1681")]; - tensor new_kv_unnorm = add(x = var_1679, y = var_1681)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_1504)[name = tensor("new_scale")]; - tensor const_37 = const()[name = tensor("const_37"), val = tensor(0x1.fffffep+127)]; - tensor clip_20 = clip(alpha = var_1362, beta = const_37, x = new_scale)[name = tensor("clip_20")]; + tensor v_masked = mul(x = v_17, y = var_1284)[name = tensor("v_masked")]; + tensor var_1467 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1467")]; + tensor var_1469_transpose_x_1 = const()[name = tensor("op_1469_transpose_x_1"), val = tensor(true)]; + tensor var_1469_transpose_y_1 = const()[name = tensor("op_1469_transpose_y_1"), val = tensor(false)]; + tensor var_1469 = matmul(transpose_x = var_1469_transpose_x_1, transpose_y = var_1469_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1469")]; + tensor new_kv_unnorm = add(x = var_1467, y = var_1469)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1292)[name = tensor("new_scale")]; + tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; + tensor clip_20 = clip(alpha = var_1149, beta = const_33, x = new_scale)[name = tensor("clip_20")]; tensor sqrt_new_scale = sqrt(x = clip_20)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_1690_perm_0 = const()[name = tensor("op_1690_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1478_perm_0 = const()[name = tensor("op_1478_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_1690 = transpose(perm = var_1690_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1359, x = var_1690)[name = tensor("out_33")]; - tensor var_1694 = const()[name = tensor("op_1694"), val = tensor([12, 2, 256])]; - tensor out = reshape(shape = var_1694, x = out_33)[name = tensor("out")]; - tensor var_1696 = silu(x = input_227)[name = tensor("op_1696")]; - tensor input_229 = mul(x = var_1696, y = out)[name = tensor("input_229")]; - tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_229)[name = tensor("linear_51")]; - tensor input_231 = add(x = x, y = ret_out)[name = tensor("input_231")]; + tensor var_1478 = transpose(perm = var_1478_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1146, x = var_1478)[name = tensor("out_33")]; + tensor var_1482 = const()[name = tensor("op_1482"), val = tensor([12, 2, 256])]; + tensor out = reshape(shape = var_1482, x = out_33)[name = tensor("out")]; + tensor var_1484 = silu(x = input_187)[name = tensor("op_1484")]; + tensor input_189 = mul(x = var_1484, y = out)[name = tensor("input_189")]; + tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; + tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1357, gamma = decoder_norm11_1_weight, x = input_231)[name = tensor("xt_5")]; - tensor var_1706 = const()[name = tensor("op_1706"), val = tensor([1, 12, 2, 256])]; - tensor var_1707 = reshape(shape = var_1706, x = xt_5)[name = tensor("op_1707")]; - tensor var_1708_perm_0 = const()[name = tensor("op_1708_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1711 = const()[name = tensor("op_1711"), val = tensor([2, 12, 256])]; - tensor var_1708 = transpose(perm = var_1708_perm_0, x = var_1707)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_1711, x = var_1708)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1144, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1494 = const()[name = tensor("op_1494"), val = tensor([1, 12, 2, 256])]; + tensor var_1495 = reshape(shape = var_1494, x = xt_5)[name = tensor("op_1495")]; + tensor var_1496_perm_0 = const()[name = tensor("op_1496_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1499 = const()[name = tensor("op_1499"), val = tensor([2, 12, 256])]; + tensor var_1496 = transpose(perm = var_1496_perm_0, x = var_1495)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1499, x = var_1496)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_1734 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_9 = const()[name = tensor("concat_9"), val = tensor([12, 2, 3, 256])]; - tensor var_1736 = reshape(shape = concat_9, x = var_1734)[name = tensor("op_1736")]; - tensor var_1737_axes_0 = const()[name = tensor("op_1737_axes_0"), val = tensor([0])]; - tensor var_1737 = expand_dims(axes = var_1737_axes_0, x = var_1736)[name = tensor("op_1737")]; - tensor var_1738_perm_0 = const()[name = tensor("op_1738_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1739_axes_0 = const()[name = tensor("op_1739_axes_0"), val = tensor([-2])]; - tensor var_1738 = transpose(perm = var_1738_perm_0, x = var_1737)[name = tensor("transpose_8")]; - tensor var_1739 = squeeze(axes = var_1739_axes_0, x = var_1738)[name = tensor("op_1739")]; + tensor var_1522 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_10 = const()[name = tensor("concat_10"), val = tensor([12, 2, 3, 256])]; + tensor var_1524 = reshape(shape = concat_10, x = var_1522)[name = tensor("op_1524")]; + tensor var_1525_axes_0 = const()[name = tensor("op_1525_axes_0"), val = tensor([0])]; + tensor var_1525 = expand_dims(axes = var_1525_axes_0, x = var_1524)[name = tensor("op_1525")]; + tensor var_1526_perm_0 = const()[name = tensor("op_1526_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1527_axes_0 = const()[name = tensor("op_1527_axes_0"), val = tensor([-2])]; + tensor var_1526 = transpose(perm = var_1526_perm_0, x = var_1525)[name = tensor("transpose_8")]; + tensor var_1527 = squeeze(axes = var_1527_axes_0, x = var_1526)[name = tensor("op_1527")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 12, 2, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1739)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1527)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 12, 2, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1739)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1527)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 12, 2, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1739)[name = tensor("v_19")]; - tensor var_1747 = const()[name = tensor("op_1747"), val = tensor([12, 8, 64])]; - tensor var_1748 = reshape(shape = var_1747, x = q_19)[name = tensor("op_1748")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1527)[name = tensor("v_19")]; + tensor var_1535 = const()[name = tensor("op_1535"), val = tensor([12, 8, 64])]; + tensor var_1536 = reshape(shape = var_1535, x = q_19)[name = tensor("op_1536")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1754 = const()[name = tensor("op_1754"), val = tensor([12, 8, 64])]; - tensor var_1755 = reshape(shape = var_1754, x = k_19)[name = tensor("op_1755")]; + tensor var_1542 = const()[name = tensor("op_1542"), val = tensor([12, 8, 64])]; + tensor var_1543 = reshape(shape = var_1542, x = k_19)[name = tensor("op_1543")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1761 = const()[name = tensor("op_1761"), val = tensor([12, 8, 64])]; - tensor var_1762 = reshape(shape = var_1761, x = v_19)[name = tensor("op_1762")]; + tensor var_1549 = const()[name = tensor("op_1549"), val = tensor([12, 8, 64])]; + tensor var_1550 = reshape(shape = var_1549, x = v_19)[name = tensor("op_1550")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1765 = const()[name = tensor("op_1765"), val = tensor([2, 4, 12, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_1748)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_1765, x = q_21)[name = tensor("q")]; - tensor var_1767 = const()[name = tensor("op_1767"), val = tensor([2, 4, 12, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_1755)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_1767, x = k_21)[name = tensor("k")]; - tensor var_1769 = const()[name = tensor("op_1769"), val = tensor([2, 4, 12, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_1762)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_1769, x = v_21)[name = tensor("v")]; + tensor var_1553 = const()[name = tensor("op_1553"), val = tensor([2, 4, 12, 64])]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1536)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1553, x = q_21)[name = tensor("q")]; + tensor var_1555 = const()[name = tensor("op_1555"), val = tensor([2, 4, 12, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1543)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1555, x = k_21)[name = tensor("k")]; + tensor var_1557 = const()[name = tensor("op_1557"), val = tensor([2, 4, 12, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1550)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1557, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -1563,50 +1416,50 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_1772 = const()[name = tensor("op_1772"), val = tensor([2, 0, 1, 3])]; - tensor var_1777 = const()[name = tensor("op_1777"), val = tensor([24, 256])]; - tensor var_1773 = transpose(perm = var_1772, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_1777, x = var_1773)[name = tensor("attn_output_11")]; + tensor var_1560 = const()[name = tensor("op_1560"), val = tensor([2, 0, 1, 3])]; + tensor var_1565 = const()[name = tensor("op_1565"), val = tensor([24, 256])]; + tensor var_1561 = transpose(perm = var_1560, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1565, x = var_1561)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_1781 = const()[name = tensor("op_1781"), val = tensor([12, 2, 256])]; - tensor attn_output = reshape(shape = var_1781, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1569 = const()[name = tensor("op_1569"), val = tensor([12, 2, 256])]; + tensor attn_output = reshape(shape = var_1569, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; - tensor input_233 = add(x = query_5, y = sa2_out)[name = tensor("input_233")]; - tensor input_235_axes_0 = const()[name = tensor("input_235_axes_0"), val = tensor([-1])]; - tensor input_235 = layer_norm(axes = input_235_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1357, gamma = decoder_norm21_1_weight, x = input_233)[name = tensor("input_235")]; - tensor input_237 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_235)[name = tensor("linear_54")]; - tensor input_239 = relu(x = input_237)[name = tensor("input_239")]; - tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_239)[name = tensor("linear_55")]; - tensor input_241 = add(x = input_235, y = ffn_out)[name = tensor("input_241")]; + tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; + tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1144, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; + tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; + tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; + tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1357, gamma = decoder_norm22_1_weight, x = input_241)[name = tensor("xt")]; - tensor var_1801 = const()[name = tensor("op_1801"), val = tensor([1, 2, 12, 256])]; - tensor input = reshape(shape = var_1801, x = xt)[name = tensor("input")]; - tensor var_1803 = const()[name = tensor("op_1803"), val = tensor([-1])]; - tensor var_1804 = reduce_l2_norm(axes = var_1803, keep_dims = var_1360, x = input)[name = tensor("op_1804")]; - tensor const_46 = const()[name = tensor("const_46"), val = tensor(0x1.fffffep+127)]; - tensor clip_21 = clip(alpha = var_1351, beta = const_46, x = var_1804)[name = tensor("clip_21")]; - tensor var_1806 = real_div(x = input, y = clip_21)[name = tensor("op_1806")]; - tensor concat_13 = const()[name = tensor("concat_13"), val = tensor([2, 1, 256])]; - tensor reshape_0 = reshape(shape = concat_13, x = emb)[name = tensor("reshape_0")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1144, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1589 = const()[name = tensor("op_1589"), val = tensor([1, 2, 12, 256])]; + tensor input = reshape(shape = var_1589, x = xt)[name = tensor("input")]; + tensor var_1591 = const()[name = tensor("op_1591"), val = tensor([-1])]; + tensor var_1592 = reduce_l2_norm(axes = var_1591, keep_dims = var_1147, x = input)[name = tensor("op_1592")]; + tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; + tensor clip_21 = clip(alpha = var_1139, beta = const_42, x = var_1592)[name = tensor("clip_21")]; + tensor var_1594 = real_div(x = input, y = clip_21)[name = tensor("op_1594")]; + tensor concat_14 = const()[name = tensor("concat_14"), val = tensor([2, 1, 256])]; + tensor reshape_0 = reshape(shape = concat_14, x = emb)[name = tensor("reshape_0")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_14 = const()[name = tensor("concat_14"), val = tensor([2, 256, 12])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1806)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_14, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_15 = const()[name = tensor("concat_15"), val = tensor([2, 256, 12])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1594)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_15, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = reshape_0, y = reshape_1)[name = tensor("matmul_2")]; - tensor concat_17 = const()[name = tensor("concat_17"), val = tensor([1, 2, 12])]; - tensor reshape_2 = reshape(shape = concat_17, x = matmul_2)[name = tensor("reshape_2")]; + tensor concat_18 = const()[name = tensor("concat_18"), val = tensor([1, 2, 12])]; + tensor reshape_2 = reshape(shape = concat_18, x = matmul_2)[name = tensor("reshape_2")]; tensor output_begin_0 = const()[name = tensor("output_begin_0"), val = tensor([0, 0, 1])]; tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 2, 11])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = reshape_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_1810")]; - tensor var_1812_axis_0 = const()[name = tensor("op_1812_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_1812_axis_0, values = (var_1508, nkv))[name = tensor("op_1812")]; - tensor var_1814_axis_0 = const()[name = tensor("op_1814_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_1814_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1814")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1598")]; + tensor var_1600_axis_0 = const()[name = tensor("op_1600_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1600_axis_0, values = (var_1296, nkv))[name = tensor("op_1600")]; + tensor var_1602_axis_0 = const()[name = tensor("op_1602_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1602_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1602")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/DIHARD II/ls_eend_dih2_200ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/DIHARD II/ls_eend_dih2_200ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index b3f3e24b58a9943d5107929adcde6a7f8a57a7a2..8e933e6049b55e1bf10e2d58cc67e2d99806c009 100644 --- a/optimized/DIHARD II/ls_eend_dih2_200ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/DIHARD II/ls_eend_dih2_200ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c758890fc4558f10a157783265451163dffa8a397f6dce05dbcbd6cbb17d6be8 -size 228673 +oid sha256:983afd05bb34a898bd1eb3ca8cf8113849d7c3c8b9bffe5e4cc276a381f829d1 +size 205524 diff --git a/optimized/DIHARD II/ls_eend_dih2_200ms.mlpackage/Manifest.json b/optimized/DIHARD II/ls_eend_dih2_200ms.mlpackage/Manifest.json index 406b27f845c5e9e8d6e46758d778c35083d554ec..0d2aad424535e9285116b739e2d48d2b5feefebc 100644 --- a/optimized/DIHARD II/ls_eend_dih2_200ms.mlpackage/Manifest.json +++ b/optimized/DIHARD II/ls_eend_dih2_200ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "00D09D87-F2B3-4451-908B-1005D6EDED98": { + "00712470-400A-4FF6-96C8-073D68B297C5": { "author": "com.apple.CoreML", "description": "CoreML Model Weights", "name": "weights", "path": "com.apple.CoreML/weights" }, - "D809B742-F3A2-442B-882C-79657675BF34": { + "D5B4E5EA-71FD-4A5B-A52B-7AA84156C796": { "author": "com.apple.CoreML", "description": "CoreML Model Specification", "name": "model.mlmodel", "path": "com.apple.CoreML/model.mlmodel" } }, - "rootModelIdentifier": "D809B742-F3A2-442B-882C-79657675BF34" + "rootModelIdentifier": "D5B4E5EA-71FD-4A5B-A52B-7AA84156C796" } diff --git a/optimized/DIHARD II/ls_eend_dih2_300ms.mlmodelc/analytics/coremldata.bin b/optimized/DIHARD II/ls_eend_dih2_300ms.mlmodelc/analytics/coremldata.bin index fd00ecadc8eb486a0682efa8577223cdc2472df3..efac7c910a96c5baf29fb61610260c13848e5efe 100644 --- a/optimized/DIHARD II/ls_eend_dih2_300ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/DIHARD II/ls_eend_dih2_300ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2c6ebfd96cfe4f3bbbf58aea0fe8a02b7f77989285f4b788ed75d9137250ba63 +oid sha256:a98d4a049bd973fc9efa021e9d718224870070eeedf91145d976ba6b1aa74c99 size 243 diff --git a/optimized/DIHARD II/ls_eend_dih2_300ms.mlmodelc/coremldata.bin b/optimized/DIHARD II/ls_eend_dih2_300ms.mlmodelc/coremldata.bin index a545f5eb651b3cf796f0eb3278fca25e43c4b98b..5a31efe5306e6d0487af15e5dd0468a14218dec4 100644 --- a/optimized/DIHARD II/ls_eend_dih2_300ms.mlmodelc/coremldata.bin +++ b/optimized/DIHARD II/ls_eend_dih2_300ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:90b8d4ae5a2cfdbfe92c5f3dffbd6597210911e48a3bc4f7c6441508a041a64b -size 1266 +oid sha256:5d63bcf9b17a5e9aa88b1d96be81040ea79831ed507731d7423dd15333ba5dcd +size 1357 diff --git a/optimized/DIHARD II/ls_eend_dih2_300ms.mlmodelc/metadata.json b/optimized/DIHARD II/ls_eend_dih2_300ms.mlmodelc/metadata.json index 2149a2d2a0973e76c2b55833bdabfef9755473d0..109b5148308d6b5c88dda77f7842650ad44c6160 100644 --- a/optimized/DIHARD II/ls_eend_dih2_300ms.mlmodelc/metadata.json +++ b/optimized/DIHARD II/ls_eend_dih2_300ms.mlmodelc/metadata.json @@ -96,30 +96,30 @@ "Ios17.reshape" : 92, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 69, - "Split" : 12, + "Ios17.transpose" : 57, + "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 82, "Tile" : 13, - "Ios17.sliceByIndex" : 76, - "Ios16.sigmoid" : 13, + "Ios17.sliceByIndex" : 60, + "Ios16.sigmoid" : 5, "Ios16.reduceSum" : 1, "Ios17.squeeze" : 2, - "Ios17.layerNorm" : 41, - "Ios17.batchNorm" : 12, + "Ios17.layerNorm" : 33, + "Ios17.batchNorm" : 4, "Ios17.cast" : 24, "Ios17.sqrt" : 18, "Ios17.sub" : 24, - "Ios17.conv" : 37, + "Ios17.conv" : 13, "Ios17.realDiv" : 20, "Ios17.linear" : 56, "Ios17.concat" : 18, "Ios17.clip" : 30, "Ios17.greaterEqual" : 12, - "Ios16.silu" : 26, + "Ios16.silu" : 18, "Stack" : 6, "Ios16.cumsum" : 1, - "Ios17.mul" : 103 + "Ios17.mul" : 95 }, "computePrecision" : "Mixed (Float32, Int32)", "isUpdatable" : "0", @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"dih2\", \"model_label\": \"DIHARD II\", \"variant\": \"pipeline\", \"T\": 3, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"dih2\", \"model_label\": \"DIHARD II\", \"variant\": \"pipeline\", \"chunk_size\": 3, \"step_duration_ms\": 300, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/DIHARD II/ls_eend_dih2_300ms.mlmodelc/model.mil b/optimized/DIHARD II/ls_eend_dih2_300ms.mlmodelc/model.mil index 4d8299dcfef93c6b52fa3dc309c7deb5a9f033c0..002a08ea435aee3ce73f649aad1d104b06bba03d 100644 --- a/optimized/DIHARD II/ls_eend_dih2_300ms.mlmodelc/model.mil +++ b/optimized/DIHARD II/ls_eend_dih2_300ms.mlmodelc/model.mil @@ -217,6 +217,7 @@ program(1.0) tensor var_21 = const()[name = tensor("op_21"), val = tensor(0x0p+0)]; tensor var_24 = const()[name = tensor("op_24"), val = tensor(0x1.0c6f7ap-20)]; tensor var_27 = const()[name = tensor("op_27"), val = tensor(2)]; + tensor var_30 = const()[name = tensor("op_30"), val = tensor(0)]; tensor var_33 = const()[name = tensor("op_33"), val = tensor(1)]; tensor var_35 = const()[name = tensor("op_35"), val = tensor(0x1.4f8b58p-17)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(true)]; @@ -263,8 +264,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_69")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_70")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_196 = const()[name = tensor("op_196"), val = tensor([3, 1])]; tensor var_197 = reshape(shape = var_196, x = sqrt_s_t_1)[name = tensor("op_197")]; @@ -272,7 +273,7 @@ program(1.0) tensor var_199 = mul(x = qk_1, y = M_1)[name = tensor("op_199")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_68")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_199, y = v_1)[name = tensor("inner_1")]; tensor var_201_transpose_x_0 = const()[name = tensor("op_201_transpose_x_0"), val = tensor(false)]; tensor var_201_transpose_y_0 = const()[name = tensor("op_201_transpose_y_0"), val = tensor(false)]; @@ -293,7 +294,7 @@ program(1.0) tensor var_214 = real_div(x = new_kv_unnorm_1, y = var_213)[name = tensor("op_214")]; tensor var_215_perm_0 = const()[name = tensor("op_215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_67")]; + tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_24, x = var_215)[name = tensor("out_3")]; tensor var_219 = const()[name = tensor("op_219"), val = tensor([1, 3, 256])]; tensor out_5 = reshape(shape = var_219, x = out_3)[name = tensor("out_5")]; @@ -341,218 +342,142 @@ program(1.0) tensor var_252 = sub(x = var_17, y = use_roll_1)[name = tensor("op_252")]; tensor var_253 = mul(x = filled_window_1, y = var_252)[name = tensor("op_253")]; tensor var_254 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_254")]; - tensor input_21 = add(x = var_253, y = var_254)[name = tensor("input_21")]; + tensor window_3 = add(x = var_253, y = var_254)[name = tensor("window_3")]; tensor var_256 = const()[name = tensor("op_256"), val = tensor(0x1p+0)]; tensor var_257 = add(x = filled_1, y = var_256)[name = tensor("op_257")]; tensor clip_1 = clip(alpha = var_21, beta = var_16, x = var_257)[name = tensor("clip_1")]; + tensor var_260_begin_0 = const()[name = tensor("op_260_begin_0"), val = tensor([0, 1, 0])]; + tensor var_260_end_0 = const()[name = tensor("op_260_end_0"), val = tensor([1, 2, 256])]; + tensor var_260_end_mask_0 = const()[name = tensor("op_260_end_mask_0"), val = tensor([true, false, true])]; + tensor var_260 = slice_by_index(begin = var_260_begin_0, end = var_260_end_0, end_mask = var_260_end_mask_0, x = x_3)[name = tensor("op_260")]; + tensor var_263_begin_0 = const()[name = tensor("op_263_begin_0"), val = tensor([0, 1, 0])]; + tensor var_263_end_0 = const()[name = tensor("op_263_end_0"), val = tensor([1, 16, 256])]; + tensor var_263_end_mask_0 = const()[name = tensor("op_263_end_mask_0"), val = tensor([true, true, true])]; + tensor var_263 = slice_by_index(begin = var_263_begin_0, end = var_263_end_0, end_mask = var_263_end_mask_0, x = window_3)[name = tensor("op_263")]; + tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; + tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_263, var_260))[name = tensor("rolled_3")]; + tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; + tensor var_268 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_268")]; + tensor var_269_dtype_0 = const()[name = tensor("op_269_dtype_0"), val = tensor("fp32")]; + tensor var_270 = const()[name = tensor("op_270"), val = tensor([1, 16, 1])]; + tensor var_269 = cast(dtype = var_269_dtype_0, x = var_268)[name = tensor("cast_157")]; + tensor fill_mask_3 = reshape(shape = var_270, x = var_269)[name = tensor("fill_mask_3")]; + tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_260)[name = tensor("x_expanded_3")]; + tensor var_274 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_274")]; + tensor var_275 = mul(x = window_3, y = var_274)[name = tensor("op_275")]; + tensor var_276 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_276")]; + tensor filled_window_3 = add(x = var_275, y = var_276)[name = tensor("filled_window_3")]; + tensor var_278 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_278")]; + tensor var_279_dtype_0 = const()[name = tensor("op_279_dtype_0"), val = tensor("fp32")]; + tensor var_280 = const()[name = tensor("op_280"), val = tensor([1, 1, 1])]; + tensor var_279 = cast(dtype = var_279_dtype_0, x = var_278)[name = tensor("cast_156")]; + tensor use_roll_3 = reshape(shape = var_280, x = var_279)[name = tensor("use_roll_3")]; + tensor var_282 = sub(x = var_17, y = use_roll_3)[name = tensor("op_282")]; + tensor var_283 = mul(x = filled_window_3, y = var_282)[name = tensor("op_283")]; + tensor var_284 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_284")]; + tensor window_5 = add(x = var_283, y = var_284)[name = tensor("window_5")]; + tensor var_286 = const()[name = tensor("op_286"), val = tensor(0x1p+0)]; + tensor var_287 = add(x = clip_1, y = var_286)[name = tensor("op_287")]; + tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_287)[name = tensor("clip_3")]; + tensor var_290_begin_0 = const()[name = tensor("op_290_begin_0"), val = tensor([0, 2, 0])]; + tensor var_290_end_0 = const()[name = tensor("op_290_end_0"), val = tensor([1, 1, 256])]; + tensor var_290_end_mask_0 = const()[name = tensor("op_290_end_mask_0"), val = tensor([true, true, true])]; + tensor var_290 = slice_by_index(begin = var_290_begin_0, end = var_290_end_0, end_mask = var_290_end_mask_0, x = x_3)[name = tensor("op_290")]; + tensor var_293_begin_0 = const()[name = tensor("op_293_begin_0"), val = tensor([0, 1, 0])]; + tensor var_293_end_0 = const()[name = tensor("op_293_end_0"), val = tensor([1, 16, 256])]; + tensor var_293_end_mask_0 = const()[name = tensor("op_293_end_mask_0"), val = tensor([true, true, true])]; + tensor var_293 = slice_by_index(begin = var_293_begin_0, end = var_293_end_0, end_mask = var_293_end_mask_0, x = window_5)[name = tensor("op_293")]; + tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; + tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_293, var_290))[name = tensor("rolled_5")]; + tensor clip_4 = clip(alpha = var_21, beta = var_20, x = clip_3)[name = tensor("clip_4")]; + tensor var_298 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_298")]; + tensor var_299_dtype_0 = const()[name = tensor("op_299_dtype_0"), val = tensor("fp32")]; + tensor var_300 = const()[name = tensor("op_300"), val = tensor([1, 16, 1])]; + tensor var_299 = cast(dtype = var_299_dtype_0, x = var_298)[name = tensor("cast_155")]; + tensor fill_mask_5 = reshape(shape = var_300, x = var_299)[name = tensor("fill_mask_5")]; + tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_290)[name = tensor("x_expanded_5")]; + tensor var_304 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_304")]; + tensor var_305 = mul(x = window_5, y = var_304)[name = tensor("op_305")]; + tensor var_306 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_306")]; + tensor filled_window_5 = add(x = var_305, y = var_306)[name = tensor("filled_window_5")]; + tensor var_308 = greater_equal(x = clip_3, y = var_16)[name = tensor("op_308")]; + tensor var_309_dtype_0 = const()[name = tensor("op_309_dtype_0"), val = tensor("fp32")]; + tensor var_310 = const()[name = tensor("op_310"), val = tensor([1, 1, 1])]; + tensor var_309 = cast(dtype = var_309_dtype_0, x = var_308)[name = tensor("cast_154")]; + tensor use_roll_5 = reshape(shape = var_310, x = var_309)[name = tensor("use_roll_5")]; + tensor var_312 = sub(x = var_17, y = use_roll_5)[name = tensor("op_312")]; + tensor var_313 = mul(x = filled_window_5, y = var_312)[name = tensor("op_313")]; + tensor var_314 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_314")]; + tensor window_7 = add(x = var_313, y = var_314)[name = tensor("window_7")]; + tensor var_316 = const()[name = tensor("op_316"), val = tensor(0x1p+0)]; + tensor var_317 = add(x = clip_3, y = var_316)[name = tensor("op_317")]; + tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_317)[name = tensor("clip_5")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_30, interleave = input_21_interleave_0, values = (window_3, window_5, window_7))[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; - tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; + tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; tensor inputs_3_pad_type_0 = const()[name = tensor("inputs_3_pad_type_0"), val = tensor("valid")]; tensor inputs_3_strides_0 = const()[name = tensor("inputs_3_strides_0"), val = tensor([1])]; tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_66")]; - tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_278_split_sizes_0 = const()[name = tensor("op_278_split_sizes_0"), val = tensor([256, 256])]; - tensor var_278_axis_0 = const()[name = tensor("op_278_axis_0"), val = tensor(1)]; - tensor var_278_0, tensor var_278_1 = split(axis = var_278_axis_0, split_sizes = var_278_split_sizes_0, x = inputs_3)[name = tensor("op_278")]; - tensor var_280 = sigmoid(x = var_278_1)[name = tensor("op_280")]; - tensor inputs_5 = mul(x = var_278_0, y = var_280)[name = tensor("inputs_5")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; + tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; + tensor var_340_split_sizes_0 = const()[name = tensor("op_340_split_sizes_0"), val = tensor([256, 256])]; + tensor var_340_axis_0 = const()[name = tensor("op_340_axis_0"), val = tensor(1)]; + tensor var_340_0, tensor var_340_1 = split(axis = var_340_axis_0, split_sizes = var_340_split_sizes_0, x = inputs_3)[name = tensor("op_340")]; + tensor var_342 = sigmoid(x = var_340_1)[name = tensor("op_342")]; + tensor inputs_5 = mul(x = var_340_0, y = var_342)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; tensor outputs_aug_1_strides_0 = const()[name = tensor("outputs_aug_1_strides_0"), val = tensor([1])]; tensor outputs_aug_1_dilations_0 = const()[name = tensor("outputs_aug_1_dilations_0"), val = tensor([1])]; - tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; + tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; tensor input_25_begin_0 = const()[name = tensor("input_25_begin_0"), val = tensor([0, 0, 0])]; - tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([1, 256, 16])]; + tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([3, 256, 16])]; tensor input_25_end_mask_0 = const()[name = tensor("input_25_end_mask_0"), val = tensor([true, true, false])]; - tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; - tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; - tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; + tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; + tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; + tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; tensor input_29_pad_type_0 = const()[name = tensor("input_29_pad_type_0"), val = tensor("valid")]; tensor input_29_strides_0 = const()[name = tensor("input_29_strides_0"), val = tensor([1])]; tensor input_29_pad_0 = const()[name = tensor("input_29_pad_0"), val = tensor([0, 0])]; tensor input_29_dilations_0 = const()[name = tensor("input_29_dilations_0"), val = tensor([1])]; tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; - tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; + tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_311_begin_0 = const()[name = tensor("op_311_begin_0"), val = tensor([0, -1, 0])]; - tensor var_311_end_0 = const()[name = tensor("op_311_end_0"), val = tensor([1, 16, 256])]; - tensor var_311_end_mask_0 = const()[name = tensor("op_311_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_65")]; - tensor var_311 = slice_by_index(begin = var_311_begin_0, end = var_311_end_0, end_mask = var_311_end_mask_0, x = conv_out_1)[name = tensor("op_311")]; - tensor var_314_begin_0 = const()[name = tensor("op_314_begin_0"), val = tensor([0, 1, 0])]; - tensor var_314_end_0 = const()[name = tensor("op_314_end_0"), val = tensor([1, 2, 256])]; - tensor var_314_end_mask_0 = const()[name = tensor("op_314_end_mask_0"), val = tensor([true, false, true])]; - tensor var_314 = slice_by_index(begin = var_314_begin_0, end = var_314_end_0, end_mask = var_314_end_mask_0, x = x_3)[name = tensor("op_314")]; - tensor var_317_begin_0 = const()[name = tensor("op_317_begin_0"), val = tensor([0, 1, 0])]; - tensor var_317_end_0 = const()[name = tensor("op_317_end_0"), val = tensor([1, 16, 256])]; - tensor var_317_end_mask_0 = const()[name = tensor("op_317_end_mask_0"), val = tensor([true, true, true])]; - tensor var_317 = slice_by_index(begin = var_317_begin_0, end = var_317_end_0, end_mask = var_317_end_mask_0, x = input_21)[name = tensor("op_317")]; - tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_317, var_314))[name = tensor("rolled_3")]; - tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; - tensor var_322 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_322")]; - tensor var_323_dtype_0 = const()[name = tensor("op_323_dtype_0"), val = tensor("fp32")]; - tensor var_324 = const()[name = tensor("op_324"), val = tensor([1, 16, 1])]; - tensor var_323 = cast(dtype = var_323_dtype_0, x = var_322)[name = tensor("cast_157")]; - tensor fill_mask_3 = reshape(shape = var_324, x = var_323)[name = tensor("fill_mask_3")]; - tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_314)[name = tensor("x_expanded_3")]; - tensor var_328 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_328")]; - tensor var_329 = mul(x = input_21, y = var_328)[name = tensor("op_329")]; - tensor var_330 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_330")]; - tensor filled_window_3 = add(x = var_329, y = var_330)[name = tensor("filled_window_3")]; - tensor var_332 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_332")]; - tensor var_333_dtype_0 = const()[name = tensor("op_333_dtype_0"), val = tensor("fp32")]; - tensor var_334 = const()[name = tensor("op_334"), val = tensor([1, 1, 1])]; - tensor var_333 = cast(dtype = var_333_dtype_0, x = var_332)[name = tensor("cast_156")]; - tensor use_roll_3 = reshape(shape = var_334, x = var_333)[name = tensor("use_roll_3")]; - tensor var_336 = sub(x = var_17, y = use_roll_3)[name = tensor("op_336")]; - tensor var_337 = mul(x = filled_window_3, y = var_336)[name = tensor("op_337")]; - tensor var_338 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_338")]; - tensor input_31 = add(x = var_337, y = var_338)[name = tensor("input_31")]; - tensor var_340 = const()[name = tensor("op_340"), val = tensor(0x1p+0)]; - tensor var_341 = add(x = clip_1, y = var_340)[name = tensor("op_341")]; - tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_341)[name = tensor("clip_3")]; + tensor var_373_begin_0 = const()[name = tensor("op_373_begin_0"), val = tensor([0, -1, 0])]; + tensor var_373_end_0 = const()[name = tensor("op_373_end_0"), val = tensor([3, 16, 256])]; + tensor var_373_end_mask_0 = const()[name = tensor("op_373_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_373 = slice_by_index(begin = var_373_begin_0, end = var_373_end_0, end_mask = var_373_end_mask_0, x = conv_out_1)[name = tensor("op_373")]; + tensor var_375_perm_0 = const()[name = tensor("op_375_perm_0"), val = tensor([1, 0, 2])]; + tensor var_375 = transpose(perm = var_375_perm_0, x = var_373)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_375)[name = tensor("input_31")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; + tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; + tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; + tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; + tensor var_398 = const()[name = tensor("op_398"), val = tensor(0x1p-1)]; + tensor var_399 = mul(x = input_39, y = var_398)[name = tensor("op_399")]; + tensor input_41 = add(x = var_399, y = input_31)[name = tensor("input_41")]; + tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; + tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; + tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; + tensor input_45 = layer_norm(axes = input_45_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_43)[name = tensor("input_45")]; + tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; + tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; + tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; + tensor var_428 = const()[name = tensor("op_428"), val = tensor(0x1p-1)]; + tensor var_429 = mul(x = input_51, y = var_428)[name = tensor("op_429")]; + tensor input_53 = add(x = var_429, y = input_43)[name = tensor("input_53")]; tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; - tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_31)[name = tensor("x_7")]; - tensor input_33_perm_0 = const()[name = tensor("input_33_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_9_pad_type_0 = const()[name = tensor("inputs_9_pad_type_0"), val = tensor("valid")]; - tensor inputs_9_strides_0 = const()[name = tensor("inputs_9_strides_0"), val = tensor([1])]; - tensor inputs_9_pad_0 = const()[name = tensor("inputs_9_pad_0"), val = tensor([0, 0])]; - tensor inputs_9_dilations_0 = const()[name = tensor("inputs_9_dilations_0"), val = tensor([1])]; - tensor inputs_9_groups_0 = const()[name = tensor("inputs_9_groups_0"), val = tensor(1)]; - tensor input_33 = transpose(perm = input_33_perm_0, x = x_7)[name = tensor("transpose_64")]; - tensor inputs_9 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_9_dilations_0, groups = inputs_9_groups_0, pad = inputs_9_pad_0, pad_type = inputs_9_pad_type_0, strides = inputs_9_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_33)[name = tensor("inputs_9")]; - tensor var_362_split_sizes_0 = const()[name = tensor("op_362_split_sizes_0"), val = tensor([256, 256])]; - tensor var_362_axis_0 = const()[name = tensor("op_362_axis_0"), val = tensor(1)]; - tensor var_362_0, tensor var_362_1 = split(axis = var_362_axis_0, split_sizes = var_362_split_sizes_0, x = inputs_9)[name = tensor("op_362")]; - tensor var_364 = sigmoid(x = var_362_1)[name = tensor("op_364")]; - tensor inputs_11 = mul(x = var_362_0, y = var_364)[name = tensor("inputs_11")]; - tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; - tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; - tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; - tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_11)[name = tensor("outputs_aug_3")]; - tensor input_35_begin_0 = const()[name = tensor("input_35_begin_0"), val = tensor([0, 0, 0])]; - tensor input_35_end_0 = const()[name = tensor("input_35_end_0"), val = tensor([1, 256, 16])]; - tensor input_35_end_mask_0 = const()[name = tensor("input_35_end_mask_0"), val = tensor([true, true, false])]; - tensor input_35 = slice_by_index(begin = input_35_begin_0, end = input_35_end_0, end_mask = input_35_end_mask_0, x = outputs_aug_3)[name = tensor("input_35")]; - tensor inputs_13 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_35)[name = tensor("inputs_13")]; - tensor input_37 = silu(x = inputs_13)[name = tensor("input_37")]; - tensor input_39_pad_type_0 = const()[name = tensor("input_39_pad_type_0"), val = tensor("valid")]; - tensor input_39_strides_0 = const()[name = tensor("input_39_strides_0"), val = tensor([1])]; - tensor input_39_pad_0 = const()[name = tensor("input_39_pad_0"), val = tensor([0, 0])]; - tensor input_39_dilations_0 = const()[name = tensor("input_39_dilations_0"), val = tensor([1])]; - tensor input_39_groups_0 = const()[name = tensor("input_39_groups_0"), val = tensor(1)]; - tensor input_39 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_37)[name = tensor("input_39")]; - tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_395_begin_0 = const()[name = tensor("op_395_begin_0"), val = tensor([0, -1, 0])]; - tensor var_395_end_0 = const()[name = tensor("op_395_end_0"), val = tensor([1, 16, 256])]; - tensor var_395_end_mask_0 = const()[name = tensor("op_395_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_39)[name = tensor("transpose_63")]; - tensor var_395 = slice_by_index(begin = var_395_begin_0, end = var_395_end_0, end_mask = var_395_end_mask_0, x = conv_out_3)[name = tensor("op_395")]; - tensor var_398_begin_0 = const()[name = tensor("op_398_begin_0"), val = tensor([0, 2, 0])]; - tensor var_398_end_0 = const()[name = tensor("op_398_end_0"), val = tensor([1, 1, 256])]; - tensor var_398_end_mask_0 = const()[name = tensor("op_398_end_mask_0"), val = tensor([true, true, true])]; - tensor var_398 = slice_by_index(begin = var_398_begin_0, end = var_398_end_0, end_mask = var_398_end_mask_0, x = x_3)[name = tensor("op_398")]; - tensor var_401_begin_0 = const()[name = tensor("op_401_begin_0"), val = tensor([0, 1, 0])]; - tensor var_401_end_0 = const()[name = tensor("op_401_end_0"), val = tensor([1, 16, 256])]; - tensor var_401_end_mask_0 = const()[name = tensor("op_401_end_mask_0"), val = tensor([true, true, true])]; - tensor var_401 = slice_by_index(begin = var_401_begin_0, end = var_401_end_0, end_mask = var_401_end_mask_0, x = input_31)[name = tensor("op_401")]; - tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_401, var_398))[name = tensor("rolled_5")]; - tensor clip_4 = clip(alpha = var_21, beta = var_20, x = clip_3)[name = tensor("clip_4")]; - tensor var_406 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_406")]; - tensor var_407_dtype_0 = const()[name = tensor("op_407_dtype_0"), val = tensor("fp32")]; - tensor var_408 = const()[name = tensor("op_408"), val = tensor([1, 16, 1])]; - tensor var_407 = cast(dtype = var_407_dtype_0, x = var_406)[name = tensor("cast_155")]; - tensor fill_mask_5 = reshape(shape = var_408, x = var_407)[name = tensor("fill_mask_5")]; - tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_398)[name = tensor("x_expanded_5")]; - tensor var_412 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_412")]; - tensor var_413 = mul(x = input_31, y = var_412)[name = tensor("op_413")]; - tensor var_414 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_414")]; - tensor filled_window_5 = add(x = var_413, y = var_414)[name = tensor("filled_window_5")]; - tensor var_416 = greater_equal(x = clip_3, y = var_16)[name = tensor("op_416")]; - tensor var_417_dtype_0 = const()[name = tensor("op_417_dtype_0"), val = tensor("fp32")]; - tensor var_418 = const()[name = tensor("op_418"), val = tensor([1, 1, 1])]; - tensor var_417 = cast(dtype = var_417_dtype_0, x = var_416)[name = tensor("cast_154")]; - tensor use_roll_5 = reshape(shape = var_418, x = var_417)[name = tensor("use_roll_5")]; - tensor var_420 = sub(x = var_17, y = use_roll_5)[name = tensor("op_420")]; - tensor var_421 = mul(x = filled_window_5, y = var_420)[name = tensor("op_421")]; - tensor var_422 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_422")]; - tensor input_41 = add(x = var_421, y = var_422)[name = tensor("input_41")]; - tensor var_424 = const()[name = tensor("op_424"), val = tensor(0x1p+0)]; - tensor var_425 = add(x = clip_3, y = var_424)[name = tensor("op_425")]; - tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_425)[name = tensor("clip_5")]; - tensor x_9_axes_0 = const()[name = tensor("x_9_axes_0"), val = tensor([-1])]; - tensor x_9 = layer_norm(axes = x_9_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_41)[name = tensor("x_9")]; - tensor input_43_perm_0 = const()[name = tensor("input_43_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_15_pad_type_0 = const()[name = tensor("inputs_15_pad_type_0"), val = tensor("valid")]; - tensor inputs_15_strides_0 = const()[name = tensor("inputs_15_strides_0"), val = tensor([1])]; - tensor inputs_15_pad_0 = const()[name = tensor("inputs_15_pad_0"), val = tensor([0, 0])]; - tensor inputs_15_dilations_0 = const()[name = tensor("inputs_15_dilations_0"), val = tensor([1])]; - tensor inputs_15_groups_0 = const()[name = tensor("inputs_15_groups_0"), val = tensor(1)]; - tensor input_43 = transpose(perm = input_43_perm_0, x = x_9)[name = tensor("transpose_62")]; - tensor inputs_15 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_15_dilations_0, groups = inputs_15_groups_0, pad = inputs_15_pad_0, pad_type = inputs_15_pad_type_0, strides = inputs_15_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_43)[name = tensor("inputs_15")]; - tensor var_446_split_sizes_0 = const()[name = tensor("op_446_split_sizes_0"), val = tensor([256, 256])]; - tensor var_446_axis_0 = const()[name = tensor("op_446_axis_0"), val = tensor(1)]; - tensor var_446_0, tensor var_446_1 = split(axis = var_446_axis_0, split_sizes = var_446_split_sizes_0, x = inputs_15)[name = tensor("op_446")]; - tensor var_448 = sigmoid(x = var_446_1)[name = tensor("op_448")]; - tensor inputs_17 = mul(x = var_446_0, y = var_448)[name = tensor("inputs_17")]; - tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; - tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; - tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; - tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_17)[name = tensor("outputs_aug_5")]; - tensor input_45_begin_0 = const()[name = tensor("input_45_begin_0"), val = tensor([0, 0, 0])]; - tensor input_45_end_0 = const()[name = tensor("input_45_end_0"), val = tensor([1, 256, 16])]; - tensor input_45_end_mask_0 = const()[name = tensor("input_45_end_mask_0"), val = tensor([true, true, false])]; - tensor input_45 = slice_by_index(begin = input_45_begin_0, end = input_45_end_0, end_mask = input_45_end_mask_0, x = outputs_aug_5)[name = tensor("input_45")]; - tensor inputs_19 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_45)[name = tensor("inputs_19")]; - tensor input_47 = silu(x = inputs_19)[name = tensor("input_47")]; - tensor input_49_pad_type_0 = const()[name = tensor("input_49_pad_type_0"), val = tensor("valid")]; - tensor input_49_strides_0 = const()[name = tensor("input_49_strides_0"), val = tensor([1])]; - tensor input_49_pad_0 = const()[name = tensor("input_49_pad_0"), val = tensor([0, 0])]; - tensor input_49_dilations_0 = const()[name = tensor("input_49_dilations_0"), val = tensor([1])]; - tensor input_49_groups_0 = const()[name = tensor("input_49_groups_0"), val = tensor(1)]; - tensor input_49 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_49_dilations_0, groups = input_49_groups_0, pad = input_49_pad_0, pad_type = input_49_pad_type_0, strides = input_49_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_47)[name = tensor("input_49")]; - tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_479_begin_0 = const()[name = tensor("op_479_begin_0"), val = tensor([0, -1, 0])]; - tensor var_479_end_0 = const()[name = tensor("op_479_end_0"), val = tensor([1, 16, 256])]; - tensor var_479_end_mask_0 = const()[name = tensor("op_479_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_49)[name = tensor("transpose_61")]; - tensor var_479 = slice_by_index(begin = var_479_begin_0, end = var_479_end_0, end_mask = var_479_end_mask_0, x = conv_out_5)[name = tensor("op_479")]; - tensor var_482_interleave_0 = const()[name = tensor("op_482_interleave_0"), val = tensor(false)]; - tensor var_482 = concat(axis = var_33, interleave = var_482_interleave_0, values = (var_311, var_395, var_479))[name = tensor("op_482")]; - tensor input_51 = add(x = x_3, y = var_482)[name = tensor("input_51")]; - tensor input_53_axes_0 = const()[name = tensor("input_53_axes_0"), val = tensor([-1])]; - tensor input_53 = layer_norm(axes = input_53_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_51)[name = tensor("input_53")]; - tensor inputs_21 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_53)[name = tensor("linear_8")]; - tensor input_55 = silu(x = inputs_21)[name = tensor("input_55")]; - tensor input_59 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_55)[name = tensor("linear_9")]; - tensor var_505 = const()[name = tensor("op_505"), val = tensor(0x1p-1)]; - tensor var_506 = mul(x = input_59, y = var_505)[name = tensor("op_506")]; - tensor input_61 = add(x = var_506, y = input_51)[name = tensor("input_61")]; - tensor input_63_axes_0 = const()[name = tensor("input_63_axes_0"), val = tensor([-1])]; - tensor input_63 = layer_norm(axes = input_63_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_61)[name = tensor("input_63")]; - tensor input_65_axes_0 = const()[name = tensor("input_65_axes_0"), val = tensor([-1])]; - tensor input_65 = layer_norm(axes = input_65_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_63)[name = tensor("input_65")]; - tensor inputs_23 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_65)[name = tensor("linear_10")]; - tensor input_67 = silu(x = inputs_23)[name = tensor("input_67")]; - tensor input_71 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_67)[name = tensor("linear_11")]; - tensor var_535 = const()[name = tensor("op_535"), val = tensor(0x1p-1)]; - tensor var_536 = mul(x = input_71, y = var_535)[name = tensor("op_536")]; - tensor input_73 = add(x = var_536, y = input_63)[name = tensor("input_73")]; - tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; - tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_73)[name = tensor("x_11")]; + tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_3_end_0 = const()[name = tensor("prev_kv_3_end_0"), val = tensor([2, 1, 4, 64, 64])]; tensor prev_kv_3_end_mask_0 = const()[name = tensor("prev_kv_3_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -563,316 +488,240 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_550 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_11)[name = tensor("linear_12")]; - tensor var_551 = const()[name = tensor("op_551"), val = tensor([1, 3, 4, 64])]; - tensor var_552 = reshape(shape = var_551, x = var_550)[name = tensor("op_552")]; + tensor var_443 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_444 = const()[name = tensor("op_444"), val = tensor([1, 3, 4, 64])]; + tensor var_445 = reshape(shape = var_444, x = var_443)[name = tensor("op_445")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_556 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_11)[name = tensor("linear_13")]; - tensor var_557 = const()[name = tensor("op_557"), val = tensor(0x1p-3)]; - tensor var_558 = mul(x = var_556, y = var_557)[name = tensor("op_558")]; - tensor var_559 = const()[name = tensor("op_559"), val = tensor([1, 3, 4, 64])]; - tensor var_560 = reshape(shape = var_559, x = var_558)[name = tensor("op_560")]; + tensor var_449 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_450 = const()[name = tensor("op_450"), val = tensor(0x1p-3)]; + tensor var_451 = mul(x = var_449, y = var_450)[name = tensor("op_451")]; + tensor var_452 = const()[name = tensor("op_452"), val = tensor([1, 3, 4, 64])]; + tensor var_453 = reshape(shape = var_452, x = var_451)[name = tensor("op_453")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_564 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_11)[name = tensor("linear_14")]; - tensor var_565 = const()[name = tensor("op_565"), val = tensor([1, 3, 4, 64])]; - tensor var_566 = reshape(shape = var_565, x = var_564)[name = tensor("op_566")]; + tensor var_457 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_458 = const()[name = tensor("op_458"), val = tensor([1, 3, 4, 64])]; + tensor var_459 = reshape(shape = var_458, x = var_457)[name = tensor("op_459")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_77 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_11)[name = tensor("linear_15")]; + tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; tensor s_t_3 = add(x = prev_scale_3, y = encoder__t_index)[name = tensor("s_t_3")]; tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_560)[name = tensor("transpose_59")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_552)[name = tensor("transpose_60")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_453)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_445)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_576 = const()[name = tensor("op_576"), val = tensor([3, 1])]; - tensor var_577 = reshape(shape = var_576, x = sqrt_s_t_3)[name = tensor("op_577")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_577)[name = tensor("M_3")]; - tensor var_579 = mul(x = qk_3, y = M_3)[name = tensor("op_579")]; + tensor var_469 = const()[name = tensor("op_469"), val = tensor([3, 1])]; + tensor var_470 = reshape(shape = var_469, x = sqrt_s_t_3)[name = tensor("op_470")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_470)[name = tensor("M_3")]; + tensor var_472 = mul(x = qk_3, y = M_3)[name = tensor("op_472")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_566)[name = tensor("transpose_58")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_579, y = v_3)[name = tensor("inner_3")]; - tensor var_581_transpose_x_0 = const()[name = tensor("op_581_transpose_x_0"), val = tensor(false)]; - tensor var_581_transpose_y_0 = const()[name = tensor("op_581_transpose_y_0"), val = tensor(false)]; - tensor var_581 = matmul(transpose_x = var_581_transpose_x_0, transpose_y = var_581_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_581")]; - tensor var_582 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_582")]; - tensor var_583 = const()[name = tensor("op_583"), val = tensor([1, 1, 3, 1])]; - tensor var_584 = reshape(shape = var_583, x = var_582)[name = tensor("op_584")]; - tensor cross_3 = mul(x = var_581, y = var_584)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_459)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_472, y = v_3)[name = tensor("inner_3")]; + tensor var_474_transpose_x_0 = const()[name = tensor("op_474_transpose_x_0"), val = tensor(false)]; + tensor var_474_transpose_y_0 = const()[name = tensor("op_474_transpose_y_0"), val = tensor(false)]; + tensor var_474 = matmul(transpose_x = var_474_transpose_x_0, transpose_y = var_474_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_474")]; + tensor var_475 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_475")]; + tensor var_476 = const()[name = tensor("op_476"), val = tensor([1, 1, 3, 1])]; + tensor var_477 = reshape(shape = var_476, x = var_475)[name = tensor("op_477")]; + tensor cross_3 = mul(x = var_474, y = var_477)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_587 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_587")]; - tensor var_589_transpose_x_1 = const()[name = tensor("op_589_transpose_x_1"), val = tensor(true)]; - tensor var_589_transpose_y_1 = const()[name = tensor("op_589_transpose_y_1"), val = tensor(false)]; - tensor var_589 = matmul(transpose_x = var_589_transpose_x_1, transpose_y = var_589_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_589")]; - tensor new_kv_unnorm_3 = add(x = var_587, y = var_589)[name = tensor("new_kv_unnorm_3")]; - tensor var_591 = const()[name = tensor("op_591"), val = tensor(0x1.8p+1)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_591)[name = tensor("new_scale_3")]; - tensor var_593 = sqrt(x = new_scale_3)[name = tensor("op_593")]; - tensor var_594 = real_div(x = new_kv_unnorm_3, y = var_593)[name = tensor("op_594")]; - tensor var_595_perm_0 = const()[name = tensor("op_595_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_480 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_480")]; + tensor var_482_transpose_x_1 = const()[name = tensor("op_482_transpose_x_1"), val = tensor(true)]; + tensor var_482_transpose_y_1 = const()[name = tensor("op_482_transpose_y_1"), val = tensor(false)]; + tensor var_482 = matmul(transpose_x = var_482_transpose_x_1, transpose_y = var_482_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_482")]; + tensor new_kv_unnorm_3 = add(x = var_480, y = var_482)[name = tensor("new_kv_unnorm_3")]; + tensor var_484 = const()[name = tensor("op_484"), val = tensor(0x1.8p+1)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_484)[name = tensor("new_scale_3")]; + tensor var_486 = sqrt(x = new_scale_3)[name = tensor("op_486")]; + tensor var_487 = real_div(x = new_kv_unnorm_3, y = var_486)[name = tensor("op_487")]; + tensor var_488_perm_0 = const()[name = tensor("op_488_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_595 = transpose(perm = var_595_perm_0, x = out_7)[name = tensor("transpose_57")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_595)[name = tensor("out_9")]; - tensor var_599 = const()[name = tensor("op_599"), val = tensor([1, 3, 256])]; - tensor out_11 = reshape(shape = var_599, x = out_9)[name = tensor("out_11")]; - tensor var_601 = silu(x = input_77)[name = tensor("op_601")]; - tensor input_79 = mul(x = var_601, y = out_11)[name = tensor("input_79")]; - tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_79)[name = tensor("linear_16")]; - tensor x_13 = add(x = input_73, y = ret_out_3)[name = tensor("x_13")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor var_488 = transpose(perm = var_488_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_488)[name = tensor("out_9")]; + tensor var_492 = const()[name = tensor("op_492"), val = tensor([1, 3, 256])]; + tensor out_11 = reshape(shape = var_492, x = out_9)[name = tensor("out_11")]; + tensor var_494 = silu(x = input_57)[name = tensor("op_494")]; + tensor input_59 = mul(x = var_494, y = out_11)[name = tensor("input_59")]; + tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; + tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; + tensor window_9_begin_0 = const()[name = tensor("window_9_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_9_end_0 = const()[name = tensor("window_9_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_9_end_mask_0 = const()[name = tensor("window_9_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_9_squeeze_mask_0 = const()[name = tensor("window_9_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_9 = slice_by_index(begin = window_9_begin_0, end = window_9_end_0, end_mask = window_9_end_mask_0, squeeze_mask = window_9_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_9")]; tensor filled_7_begin_0 = const()[name = tensor("filled_7_begin_0"), val = tensor([1, 0])]; tensor filled_7_end_0 = const()[name = tensor("filled_7_end_0"), val = tensor([2, 1])]; tensor filled_7_end_mask_0 = const()[name = tensor("filled_7_end_mask_0"), val = tensor([false, true])]; tensor filled_7_squeeze_mask_0 = const()[name = tensor("filled_7_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_7 = slice_by_index(begin = filled_7_begin_0, end = filled_7_end_0, end_mask = filled_7_end_mask_0, squeeze_mask = filled_7_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_7")]; - tensor var_610_begin_0 = const()[name = tensor("op_610_begin_0"), val = tensor([0, 0, 0])]; - tensor var_610_end_0 = const()[name = tensor("op_610_end_0"), val = tensor([1, 1, 256])]; - tensor var_610_end_mask_0 = const()[name = tensor("op_610_end_mask_0"), val = tensor([true, false, true])]; - tensor var_610 = slice_by_index(begin = var_610_begin_0, end = var_610_end_0, end_mask = var_610_end_mask_0, x = x_13)[name = tensor("op_610")]; - tensor var_613_begin_0 = const()[name = tensor("op_613_begin_0"), val = tensor([0, 1, 0])]; - tensor var_613_end_0 = const()[name = tensor("op_613_end_0"), val = tensor([1, 16, 256])]; - tensor var_613_end_mask_0 = const()[name = tensor("op_613_end_mask_0"), val = tensor([true, true, true])]; - tensor var_613 = slice_by_index(begin = var_613_begin_0, end = var_613_end_0, end_mask = var_613_end_mask_0, x = window_3)[name = tensor("op_613")]; + tensor var_503_begin_0 = const()[name = tensor("op_503_begin_0"), val = tensor([0, 0, 0])]; + tensor var_503_end_0 = const()[name = tensor("op_503_end_0"), val = tensor([1, 1, 256])]; + tensor var_503_end_mask_0 = const()[name = tensor("op_503_end_mask_0"), val = tensor([true, false, true])]; + tensor var_503 = slice_by_index(begin = var_503_begin_0, end = var_503_end_0, end_mask = var_503_end_mask_0, x = x_9)[name = tensor("op_503")]; + tensor var_506_begin_0 = const()[name = tensor("op_506_begin_0"), val = tensor([0, 1, 0])]; + tensor var_506_end_0 = const()[name = tensor("op_506_end_0"), val = tensor([1, 16, 256])]; + tensor var_506_end_mask_0 = const()[name = tensor("op_506_end_mask_0"), val = tensor([true, true, true])]; + tensor var_506 = slice_by_index(begin = var_506_begin_0, end = var_506_end_0, end_mask = var_506_end_mask_0, x = window_9)[name = tensor("op_506")]; tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; - tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_613, var_610))[name = tensor("rolled_7")]; + tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_506, var_503))[name = tensor("rolled_7")]; tensor clip_6 = clip(alpha = var_21, beta = var_20, x = filled_7)[name = tensor("clip_6")]; - tensor var_618 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_618")]; - tensor var_619_dtype_0 = const()[name = tensor("op_619_dtype_0"), val = tensor("fp32")]; - tensor var_620 = const()[name = tensor("op_620"), val = tensor([1, 16, 1])]; - tensor var_619 = cast(dtype = var_619_dtype_0, x = var_618)[name = tensor("cast_153")]; - tensor fill_mask_7 = reshape(shape = var_620, x = var_619)[name = tensor("fill_mask_7")]; + tensor var_511 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_511")]; + tensor var_512_dtype_0 = const()[name = tensor("op_512_dtype_0"), val = tensor("fp32")]; + tensor var_513 = const()[name = tensor("op_513"), val = tensor([1, 16, 1])]; + tensor var_512 = cast(dtype = var_512_dtype_0, x = var_511)[name = tensor("cast_153")]; + tensor fill_mask_7 = reshape(shape = var_513, x = var_512)[name = tensor("fill_mask_7")]; tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_610)[name = tensor("x_expanded_7")]; - tensor var_624 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_624")]; - tensor var_625 = mul(x = window_3, y = var_624)[name = tensor("op_625")]; - tensor var_626 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_626")]; - tensor filled_window_7 = add(x = var_625, y = var_626)[name = tensor("filled_window_7")]; - tensor var_628 = greater_equal(x = filled_7, y = var_16)[name = tensor("op_628")]; - tensor var_629_dtype_0 = const()[name = tensor("op_629_dtype_0"), val = tensor("fp32")]; - tensor var_630 = const()[name = tensor("op_630"), val = tensor([1, 1, 1])]; - tensor var_629 = cast(dtype = var_629_dtype_0, x = var_628)[name = tensor("cast_152")]; - tensor use_roll_7 = reshape(shape = var_630, x = var_629)[name = tensor("use_roll_7")]; - tensor var_632 = sub(x = var_17, y = use_roll_7)[name = tensor("op_632")]; - tensor var_633 = mul(x = filled_window_7, y = var_632)[name = tensor("op_633")]; - tensor var_634 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_634")]; - tensor input_81 = add(x = var_633, y = var_634)[name = tensor("input_81")]; - tensor var_636 = const()[name = tensor("op_636"), val = tensor(0x1p+0)]; - tensor var_637 = add(x = filled_7, y = var_636)[name = tensor("op_637")]; - tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_637)[name = tensor("clip_7")]; - tensor x_15_axes_0 = const()[name = tensor("x_15_axes_0"), val = tensor([-1])]; - tensor x_15 = layer_norm(axes = x_15_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_81)[name = tensor("x_15")]; - tensor input_83_perm_0 = const()[name = tensor("input_83_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_25_pad_type_0 = const()[name = tensor("inputs_25_pad_type_0"), val = tensor("valid")]; - tensor inputs_25_strides_0 = const()[name = tensor("inputs_25_strides_0"), val = tensor([1])]; - tensor inputs_25_pad_0 = const()[name = tensor("inputs_25_pad_0"), val = tensor([0, 0])]; - tensor inputs_25_dilations_0 = const()[name = tensor("inputs_25_dilations_0"), val = tensor([1])]; - tensor inputs_25_groups_0 = const()[name = tensor("inputs_25_groups_0"), val = tensor(1)]; - tensor input_83 = transpose(perm = input_83_perm_0, x = x_15)[name = tensor("transpose_56")]; - tensor inputs_25 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_25_dilations_0, groups = inputs_25_groups_0, pad = inputs_25_pad_0, pad_type = inputs_25_pad_type_0, strides = inputs_25_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_83)[name = tensor("inputs_25")]; - tensor var_658_split_sizes_0 = const()[name = tensor("op_658_split_sizes_0"), val = tensor([256, 256])]; - tensor var_658_axis_0 = const()[name = tensor("op_658_axis_0"), val = tensor(1)]; - tensor var_658_0, tensor var_658_1 = split(axis = var_658_axis_0, split_sizes = var_658_split_sizes_0, x = inputs_25)[name = tensor("op_658")]; - tensor var_660 = sigmoid(x = var_658_1)[name = tensor("op_660")]; - tensor inputs_27 = mul(x = var_658_0, y = var_660)[name = tensor("inputs_27")]; - tensor outputs_aug_7_pad_type_0 = const()[name = tensor("outputs_aug_7_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_7_pad_0 = const()[name = tensor("outputs_aug_7_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_7_groups_0 = const()[name = tensor("outputs_aug_7_groups_0"), val = tensor(256)]; - tensor outputs_aug_7_strides_0 = const()[name = tensor("outputs_aug_7_strides_0"), val = tensor([1])]; - tensor outputs_aug_7_dilations_0 = const()[name = tensor("outputs_aug_7_dilations_0"), val = tensor([1])]; - tensor outputs_aug_7 = conv(dilations = outputs_aug_7_dilations_0, groups = outputs_aug_7_groups_0, pad = outputs_aug_7_pad_0, pad_type = outputs_aug_7_pad_type_0, strides = outputs_aug_7_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_27)[name = tensor("outputs_aug_7")]; - tensor input_85_begin_0 = const()[name = tensor("input_85_begin_0"), val = tensor([0, 0, 0])]; - tensor input_85_end_0 = const()[name = tensor("input_85_end_0"), val = tensor([1, 256, 16])]; - tensor input_85_end_mask_0 = const()[name = tensor("input_85_end_mask_0"), val = tensor([true, true, false])]; - tensor input_85 = slice_by_index(begin = input_85_begin_0, end = input_85_end_0, end_mask = input_85_end_mask_0, x = outputs_aug_7)[name = tensor("input_85")]; - tensor inputs_29 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_85)[name = tensor("inputs_29")]; - tensor input_87 = silu(x = inputs_29)[name = tensor("input_87")]; - tensor input_89_pad_type_0 = const()[name = tensor("input_89_pad_type_0"), val = tensor("valid")]; - tensor input_89_strides_0 = const()[name = tensor("input_89_strides_0"), val = tensor([1])]; - tensor input_89_pad_0 = const()[name = tensor("input_89_pad_0"), val = tensor([0, 0])]; - tensor input_89_dilations_0 = const()[name = tensor("input_89_dilations_0"), val = tensor([1])]; - tensor input_89_groups_0 = const()[name = tensor("input_89_groups_0"), val = tensor(1)]; - tensor input_89 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_89_dilations_0, groups = input_89_groups_0, pad = input_89_pad_0, pad_type = input_89_pad_type_0, strides = input_89_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_87)[name = tensor("input_89")]; - tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_691_begin_0 = const()[name = tensor("op_691_begin_0"), val = tensor([0, -1, 0])]; - tensor var_691_end_0 = const()[name = tensor("op_691_end_0"), val = tensor([1, 16, 256])]; - tensor var_691_end_mask_0 = const()[name = tensor("op_691_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_89)[name = tensor("transpose_55")]; - tensor var_691 = slice_by_index(begin = var_691_begin_0, end = var_691_end_0, end_mask = var_691_end_mask_0, x = conv_out_7)[name = tensor("op_691")]; - tensor var_694_begin_0 = const()[name = tensor("op_694_begin_0"), val = tensor([0, 1, 0])]; - tensor var_694_end_0 = const()[name = tensor("op_694_end_0"), val = tensor([1, 2, 256])]; - tensor var_694_end_mask_0 = const()[name = tensor("op_694_end_mask_0"), val = tensor([true, false, true])]; - tensor var_694 = slice_by_index(begin = var_694_begin_0, end = var_694_end_0, end_mask = var_694_end_mask_0, x = x_13)[name = tensor("op_694")]; - tensor var_697_begin_0 = const()[name = tensor("op_697_begin_0"), val = tensor([0, 1, 0])]; - tensor var_697_end_0 = const()[name = tensor("op_697_end_0"), val = tensor([1, 16, 256])]; - tensor var_697_end_mask_0 = const()[name = tensor("op_697_end_mask_0"), val = tensor([true, true, true])]; - tensor var_697 = slice_by_index(begin = var_697_begin_0, end = var_697_end_0, end_mask = var_697_end_mask_0, x = input_81)[name = tensor("op_697")]; + tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_503)[name = tensor("x_expanded_7")]; + tensor var_517 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_517")]; + tensor var_518 = mul(x = window_9, y = var_517)[name = tensor("op_518")]; + tensor var_519 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_519")]; + tensor filled_window_7 = add(x = var_518, y = var_519)[name = tensor("filled_window_7")]; + tensor var_521 = greater_equal(x = filled_7, y = var_16)[name = tensor("op_521")]; + tensor var_522_dtype_0 = const()[name = tensor("op_522_dtype_0"), val = tensor("fp32")]; + tensor var_523 = const()[name = tensor("op_523"), val = tensor([1, 1, 1])]; + tensor var_522 = cast(dtype = var_522_dtype_0, x = var_521)[name = tensor("cast_152")]; + tensor use_roll_7 = reshape(shape = var_523, x = var_522)[name = tensor("use_roll_7")]; + tensor var_525 = sub(x = var_17, y = use_roll_7)[name = tensor("op_525")]; + tensor var_526 = mul(x = filled_window_7, y = var_525)[name = tensor("op_526")]; + tensor var_527 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_527")]; + tensor window_11 = add(x = var_526, y = var_527)[name = tensor("window_11")]; + tensor var_529 = const()[name = tensor("op_529"), val = tensor(0x1p+0)]; + tensor var_530 = add(x = filled_7, y = var_529)[name = tensor("op_530")]; + tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_530)[name = tensor("clip_7")]; + tensor var_533_begin_0 = const()[name = tensor("op_533_begin_0"), val = tensor([0, 1, 0])]; + tensor var_533_end_0 = const()[name = tensor("op_533_end_0"), val = tensor([1, 2, 256])]; + tensor var_533_end_mask_0 = const()[name = tensor("op_533_end_mask_0"), val = tensor([true, false, true])]; + tensor var_533 = slice_by_index(begin = var_533_begin_0, end = var_533_end_0, end_mask = var_533_end_mask_0, x = x_9)[name = tensor("op_533")]; + tensor var_536_begin_0 = const()[name = tensor("op_536_begin_0"), val = tensor([0, 1, 0])]; + tensor var_536_end_0 = const()[name = tensor("op_536_end_0"), val = tensor([1, 16, 256])]; + tensor var_536_end_mask_0 = const()[name = tensor("op_536_end_mask_0"), val = tensor([true, true, true])]; + tensor var_536 = slice_by_index(begin = var_536_begin_0, end = var_536_end_0, end_mask = var_536_end_mask_0, x = window_11)[name = tensor("op_536")]; tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; - tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_697, var_694))[name = tensor("rolled_9")]; + tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_536, var_533))[name = tensor("rolled_9")]; tensor clip_8 = clip(alpha = var_21, beta = var_20, x = clip_7)[name = tensor("clip_8")]; - tensor var_702 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_702")]; - tensor var_703_dtype_0 = const()[name = tensor("op_703_dtype_0"), val = tensor("fp32")]; - tensor var_704 = const()[name = tensor("op_704"), val = tensor([1, 16, 1])]; - tensor var_703 = cast(dtype = var_703_dtype_0, x = var_702)[name = tensor("cast_151")]; - tensor fill_mask_9 = reshape(shape = var_704, x = var_703)[name = tensor("fill_mask_9")]; + tensor var_541 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_541")]; + tensor var_542_dtype_0 = const()[name = tensor("op_542_dtype_0"), val = tensor("fp32")]; + tensor var_543 = const()[name = tensor("op_543"), val = tensor([1, 16, 1])]; + tensor var_542 = cast(dtype = var_542_dtype_0, x = var_541)[name = tensor("cast_151")]; + tensor fill_mask_9 = reshape(shape = var_543, x = var_542)[name = tensor("fill_mask_9")]; tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_694)[name = tensor("x_expanded_9")]; - tensor var_708 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_708")]; - tensor var_709 = mul(x = input_81, y = var_708)[name = tensor("op_709")]; - tensor var_710 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_710")]; - tensor filled_window_9 = add(x = var_709, y = var_710)[name = tensor("filled_window_9")]; - tensor var_712 = greater_equal(x = clip_7, y = var_16)[name = tensor("op_712")]; - tensor var_713_dtype_0 = const()[name = tensor("op_713_dtype_0"), val = tensor("fp32")]; - tensor var_714 = const()[name = tensor("op_714"), val = tensor([1, 1, 1])]; - tensor var_713 = cast(dtype = var_713_dtype_0, x = var_712)[name = tensor("cast_150")]; - tensor use_roll_9 = reshape(shape = var_714, x = var_713)[name = tensor("use_roll_9")]; - tensor var_716 = sub(x = var_17, y = use_roll_9)[name = tensor("op_716")]; - tensor var_717 = mul(x = filled_window_9, y = var_716)[name = tensor("op_717")]; - tensor var_718 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_718")]; - tensor input_91 = add(x = var_717, y = var_718)[name = tensor("input_91")]; - tensor var_720 = const()[name = tensor("op_720"), val = tensor(0x1p+0)]; - tensor var_721 = add(x = clip_7, y = var_720)[name = tensor("op_721")]; - tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_721)[name = tensor("clip_9")]; - tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; - tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_91)[name = tensor("x_17")]; - tensor input_93_perm_0 = const()[name = tensor("input_93_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_31_pad_type_0 = const()[name = tensor("inputs_31_pad_type_0"), val = tensor("valid")]; - tensor inputs_31_strides_0 = const()[name = tensor("inputs_31_strides_0"), val = tensor([1])]; - tensor inputs_31_pad_0 = const()[name = tensor("inputs_31_pad_0"), val = tensor([0, 0])]; - tensor inputs_31_dilations_0 = const()[name = tensor("inputs_31_dilations_0"), val = tensor([1])]; - tensor inputs_31_groups_0 = const()[name = tensor("inputs_31_groups_0"), val = tensor(1)]; - tensor input_93 = transpose(perm = input_93_perm_0, x = x_17)[name = tensor("transpose_54")]; - tensor inputs_31 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_31_dilations_0, groups = inputs_31_groups_0, pad = inputs_31_pad_0, pad_type = inputs_31_pad_type_0, strides = inputs_31_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_93)[name = tensor("inputs_31")]; - tensor var_742_split_sizes_0 = const()[name = tensor("op_742_split_sizes_0"), val = tensor([256, 256])]; - tensor var_742_axis_0 = const()[name = tensor("op_742_axis_0"), val = tensor(1)]; - tensor var_742_0, tensor var_742_1 = split(axis = var_742_axis_0, split_sizes = var_742_split_sizes_0, x = inputs_31)[name = tensor("op_742")]; - tensor var_744 = sigmoid(x = var_742_1)[name = tensor("op_744")]; - tensor inputs_33 = mul(x = var_742_0, y = var_744)[name = tensor("inputs_33")]; - tensor outputs_aug_9_pad_type_0 = const()[name = tensor("outputs_aug_9_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_9_pad_0 = const()[name = tensor("outputs_aug_9_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_9_groups_0 = const()[name = tensor("outputs_aug_9_groups_0"), val = tensor(256)]; - tensor outputs_aug_9_strides_0 = const()[name = tensor("outputs_aug_9_strides_0"), val = tensor([1])]; - tensor outputs_aug_9_dilations_0 = const()[name = tensor("outputs_aug_9_dilations_0"), val = tensor([1])]; - tensor outputs_aug_9 = conv(dilations = outputs_aug_9_dilations_0, groups = outputs_aug_9_groups_0, pad = outputs_aug_9_pad_0, pad_type = outputs_aug_9_pad_type_0, strides = outputs_aug_9_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_33)[name = tensor("outputs_aug_9")]; - tensor input_95_begin_0 = const()[name = tensor("input_95_begin_0"), val = tensor([0, 0, 0])]; - tensor input_95_end_0 = const()[name = tensor("input_95_end_0"), val = tensor([1, 256, 16])]; - tensor input_95_end_mask_0 = const()[name = tensor("input_95_end_mask_0"), val = tensor([true, true, false])]; - tensor input_95 = slice_by_index(begin = input_95_begin_0, end = input_95_end_0, end_mask = input_95_end_mask_0, x = outputs_aug_9)[name = tensor("input_95")]; - tensor inputs_35 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_95)[name = tensor("inputs_35")]; - tensor input_97 = silu(x = inputs_35)[name = tensor("input_97")]; - tensor input_99_pad_type_0 = const()[name = tensor("input_99_pad_type_0"), val = tensor("valid")]; - tensor input_99_strides_0 = const()[name = tensor("input_99_strides_0"), val = tensor([1])]; - tensor input_99_pad_0 = const()[name = tensor("input_99_pad_0"), val = tensor([0, 0])]; - tensor input_99_dilations_0 = const()[name = tensor("input_99_dilations_0"), val = tensor([1])]; - tensor input_99_groups_0 = const()[name = tensor("input_99_groups_0"), val = tensor(1)]; - tensor input_99 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_99_dilations_0, groups = input_99_groups_0, pad = input_99_pad_0, pad_type = input_99_pad_type_0, strides = input_99_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_97)[name = tensor("input_99")]; - tensor conv_out_9_perm_0 = const()[name = tensor("conv_out_9_perm_0"), val = tensor([0, 2, 1])]; - tensor var_775_begin_0 = const()[name = tensor("op_775_begin_0"), val = tensor([0, -1, 0])]; - tensor var_775_end_0 = const()[name = tensor("op_775_end_0"), val = tensor([1, 16, 256])]; - tensor var_775_end_mask_0 = const()[name = tensor("op_775_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_9 = transpose(perm = conv_out_9_perm_0, x = input_99)[name = tensor("transpose_53")]; - tensor var_775 = slice_by_index(begin = var_775_begin_0, end = var_775_end_0, end_mask = var_775_end_mask_0, x = conv_out_9)[name = tensor("op_775")]; - tensor var_778_begin_0 = const()[name = tensor("op_778_begin_0"), val = tensor([0, 2, 0])]; - tensor var_778_end_0 = const()[name = tensor("op_778_end_0"), val = tensor([1, 1, 256])]; - tensor var_778_end_mask_0 = const()[name = tensor("op_778_end_mask_0"), val = tensor([true, true, true])]; - tensor var_778 = slice_by_index(begin = var_778_begin_0, end = var_778_end_0, end_mask = var_778_end_mask_0, x = x_13)[name = tensor("op_778")]; - tensor var_781_begin_0 = const()[name = tensor("op_781_begin_0"), val = tensor([0, 1, 0])]; - tensor var_781_end_0 = const()[name = tensor("op_781_end_0"), val = tensor([1, 16, 256])]; - tensor var_781_end_mask_0 = const()[name = tensor("op_781_end_mask_0"), val = tensor([true, true, true])]; - tensor var_781 = slice_by_index(begin = var_781_begin_0, end = var_781_end_0, end_mask = var_781_end_mask_0, x = input_91)[name = tensor("op_781")]; + tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_533)[name = tensor("x_expanded_9")]; + tensor var_547 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_547")]; + tensor var_548 = mul(x = window_11, y = var_547)[name = tensor("op_548")]; + tensor var_549 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_549")]; + tensor filled_window_9 = add(x = var_548, y = var_549)[name = tensor("filled_window_9")]; + tensor var_551 = greater_equal(x = clip_7, y = var_16)[name = tensor("op_551")]; + tensor var_552_dtype_0 = const()[name = tensor("op_552_dtype_0"), val = tensor("fp32")]; + tensor var_553 = const()[name = tensor("op_553"), val = tensor([1, 1, 1])]; + tensor var_552 = cast(dtype = var_552_dtype_0, x = var_551)[name = tensor("cast_150")]; + tensor use_roll_9 = reshape(shape = var_553, x = var_552)[name = tensor("use_roll_9")]; + tensor var_555 = sub(x = var_17, y = use_roll_9)[name = tensor("op_555")]; + tensor var_556 = mul(x = filled_window_9, y = var_555)[name = tensor("op_556")]; + tensor var_557 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_557")]; + tensor window_13 = add(x = var_556, y = var_557)[name = tensor("window_13")]; + tensor var_559 = const()[name = tensor("op_559"), val = tensor(0x1p+0)]; + tensor var_560 = add(x = clip_7, y = var_559)[name = tensor("op_560")]; + tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_560)[name = tensor("clip_9")]; + tensor var_563_begin_0 = const()[name = tensor("op_563_begin_0"), val = tensor([0, 2, 0])]; + tensor var_563_end_0 = const()[name = tensor("op_563_end_0"), val = tensor([1, 1, 256])]; + tensor var_563_end_mask_0 = const()[name = tensor("op_563_end_mask_0"), val = tensor([true, true, true])]; + tensor var_563 = slice_by_index(begin = var_563_begin_0, end = var_563_end_0, end_mask = var_563_end_mask_0, x = x_9)[name = tensor("op_563")]; + tensor var_566_begin_0 = const()[name = tensor("op_566_begin_0"), val = tensor([0, 1, 0])]; + tensor var_566_end_0 = const()[name = tensor("op_566_end_0"), val = tensor([1, 16, 256])]; + tensor var_566_end_mask_0 = const()[name = tensor("op_566_end_mask_0"), val = tensor([true, true, true])]; + tensor var_566 = slice_by_index(begin = var_566_begin_0, end = var_566_end_0, end_mask = var_566_end_mask_0, x = window_13)[name = tensor("op_566")]; tensor rolled_11_interleave_0 = const()[name = tensor("rolled_11_interleave_0"), val = tensor(false)]; - tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_781, var_778))[name = tensor("rolled_11")]; + tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_566, var_563))[name = tensor("rolled_11")]; tensor clip_10 = clip(alpha = var_21, beta = var_20, x = clip_9)[name = tensor("clip_10")]; - tensor var_786 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_786")]; - tensor var_787_dtype_0 = const()[name = tensor("op_787_dtype_0"), val = tensor("fp32")]; - tensor var_788 = const()[name = tensor("op_788"), val = tensor([1, 16, 1])]; - tensor var_787 = cast(dtype = var_787_dtype_0, x = var_786)[name = tensor("cast_149")]; - tensor fill_mask_11 = reshape(shape = var_788, x = var_787)[name = tensor("fill_mask_11")]; + tensor var_571 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_571")]; + tensor var_572_dtype_0 = const()[name = tensor("op_572_dtype_0"), val = tensor("fp32")]; + tensor var_573 = const()[name = tensor("op_573"), val = tensor([1, 16, 1])]; + tensor var_572 = cast(dtype = var_572_dtype_0, x = var_571)[name = tensor("cast_149")]; + tensor fill_mask_11 = reshape(shape = var_573, x = var_572)[name = tensor("fill_mask_11")]; tensor x_expanded_11_reps_0 = const()[name = tensor("x_expanded_11_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_778)[name = tensor("x_expanded_11")]; - tensor var_792 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_792")]; - tensor var_793 = mul(x = input_91, y = var_792)[name = tensor("op_793")]; - tensor var_794 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_794")]; - tensor filled_window_11 = add(x = var_793, y = var_794)[name = tensor("filled_window_11")]; - tensor var_796 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_796")]; - tensor var_797_dtype_0 = const()[name = tensor("op_797_dtype_0"), val = tensor("fp32")]; - tensor var_798 = const()[name = tensor("op_798"), val = tensor([1, 1, 1])]; - tensor var_797 = cast(dtype = var_797_dtype_0, x = var_796)[name = tensor("cast_148")]; - tensor use_roll_11 = reshape(shape = var_798, x = var_797)[name = tensor("use_roll_11")]; - tensor var_800 = sub(x = var_17, y = use_roll_11)[name = tensor("op_800")]; - tensor var_801 = mul(x = filled_window_11, y = var_800)[name = tensor("op_801")]; - tensor var_802 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_802")]; - tensor input_101 = add(x = var_801, y = var_802)[name = tensor("input_101")]; - tensor var_804 = const()[name = tensor("op_804"), val = tensor(0x1p+0)]; - tensor var_805 = add(x = clip_9, y = var_804)[name = tensor("op_805")]; - tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_805)[name = tensor("clip_11")]; - tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; - tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_101)[name = tensor("x_19")]; - tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_37_pad_type_0 = const()[name = tensor("inputs_37_pad_type_0"), val = tensor("valid")]; - tensor inputs_37_strides_0 = const()[name = tensor("inputs_37_strides_0"), val = tensor([1])]; - tensor inputs_37_pad_0 = const()[name = tensor("inputs_37_pad_0"), val = tensor([0, 0])]; - tensor inputs_37_dilations_0 = const()[name = tensor("inputs_37_dilations_0"), val = tensor([1])]; - tensor inputs_37_groups_0 = const()[name = tensor("inputs_37_groups_0"), val = tensor(1)]; - tensor input_103 = transpose(perm = input_103_perm_0, x = x_19)[name = tensor("transpose_52")]; - tensor inputs_37 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_37_dilations_0, groups = inputs_37_groups_0, pad = inputs_37_pad_0, pad_type = inputs_37_pad_type_0, strides = inputs_37_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_37")]; - tensor var_826_split_sizes_0 = const()[name = tensor("op_826_split_sizes_0"), val = tensor([256, 256])]; - tensor var_826_axis_0 = const()[name = tensor("op_826_axis_0"), val = tensor(1)]; - tensor var_826_0, tensor var_826_1 = split(axis = var_826_axis_0, split_sizes = var_826_split_sizes_0, x = inputs_37)[name = tensor("op_826")]; - tensor var_828 = sigmoid(x = var_826_1)[name = tensor("op_828")]; - tensor inputs_39 = mul(x = var_826_0, y = var_828)[name = tensor("inputs_39")]; - tensor outputs_aug_11_pad_type_0 = const()[name = tensor("outputs_aug_11_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_11_pad_0 = const()[name = tensor("outputs_aug_11_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_11_groups_0 = const()[name = tensor("outputs_aug_11_groups_0"), val = tensor(256)]; - tensor outputs_aug_11_strides_0 = const()[name = tensor("outputs_aug_11_strides_0"), val = tensor([1])]; - tensor outputs_aug_11_dilations_0 = const()[name = tensor("outputs_aug_11_dilations_0"), val = tensor([1])]; - tensor outputs_aug_11 = conv(dilations = outputs_aug_11_dilations_0, groups = outputs_aug_11_groups_0, pad = outputs_aug_11_pad_0, pad_type = outputs_aug_11_pad_type_0, strides = outputs_aug_11_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_39)[name = tensor("outputs_aug_11")]; - tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; - tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([1, 256, 16])]; - tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; - tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_11)[name = tensor("input_105")]; - tensor inputs_41 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_105)[name = tensor("inputs_41")]; - tensor input_107 = silu(x = inputs_41)[name = tensor("input_107")]; - tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; - tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; - tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; - tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; - tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; - tensor input_109 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; - tensor conv_out_11_perm_0 = const()[name = tensor("conv_out_11_perm_0"), val = tensor([0, 2, 1])]; - tensor var_859_begin_0 = const()[name = tensor("op_859_begin_0"), val = tensor([0, -1, 0])]; - tensor var_859_end_0 = const()[name = tensor("op_859_end_0"), val = tensor([1, 16, 256])]; - tensor var_859_end_mask_0 = const()[name = tensor("op_859_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_11 = transpose(perm = conv_out_11_perm_0, x = input_109)[name = tensor("transpose_51")]; - tensor var_859 = slice_by_index(begin = var_859_begin_0, end = var_859_end_0, end_mask = var_859_end_mask_0, x = conv_out_11)[name = tensor("op_859")]; - tensor var_862_interleave_0 = const()[name = tensor("op_862_interleave_0"), val = tensor(false)]; - tensor var_862 = concat(axis = var_33, interleave = var_862_interleave_0, values = (var_691, var_775, var_859))[name = tensor("op_862")]; - tensor input_111 = add(x = x_13, y = var_862)[name = tensor("input_111")]; - tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; - tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; - tensor inputs_43 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_17")]; - tensor input_115 = silu(x = inputs_43)[name = tensor("input_115")]; - tensor input_119 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_18")]; - tensor var_885 = const()[name = tensor("op_885"), val = tensor(0x1p-1)]; - tensor var_886 = mul(x = input_119, y = var_885)[name = tensor("op_886")]; - tensor input_121 = add(x = var_886, y = input_111)[name = tensor("input_121")]; - tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; - tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_121)[name = tensor("input_123")]; - tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; - tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; - tensor inputs_45 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_19")]; - tensor input_127 = silu(x = inputs_45)[name = tensor("input_127")]; - tensor input_131 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_20")]; - tensor var_915 = const()[name = tensor("op_915"), val = tensor(0x1p-1)]; - tensor var_916 = mul(x = input_131, y = var_915)[name = tensor("op_916")]; - tensor input_133 = add(x = var_916, y = input_123)[name = tensor("input_133")]; - tensor x_21_axes_0 = const()[name = tensor("x_21_axes_0"), val = tensor([-1])]; - tensor x_21 = layer_norm(axes = x_21_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_133)[name = tensor("x_21")]; + tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_563)[name = tensor("x_expanded_11")]; + tensor var_577 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_577")]; + tensor var_578 = mul(x = window_13, y = var_577)[name = tensor("op_578")]; + tensor var_579 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_579")]; + tensor filled_window_11 = add(x = var_578, y = var_579)[name = tensor("filled_window_11")]; + tensor var_581 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_581")]; + tensor var_582_dtype_0 = const()[name = tensor("op_582_dtype_0"), val = tensor("fp32")]; + tensor var_583 = const()[name = tensor("op_583"), val = tensor([1, 1, 1])]; + tensor var_582 = cast(dtype = var_582_dtype_0, x = var_581)[name = tensor("cast_148")]; + tensor use_roll_11 = reshape(shape = var_583, x = var_582)[name = tensor("use_roll_11")]; + tensor var_585 = sub(x = var_17, y = use_roll_11)[name = tensor("op_585")]; + tensor var_586 = mul(x = filled_window_11, y = var_585)[name = tensor("op_586")]; + tensor var_587 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_587")]; + tensor window_15 = add(x = var_586, y = var_587)[name = tensor("window_15")]; + tensor var_589 = const()[name = tensor("op_589"), val = tensor(0x1p+0)]; + tensor var_590 = add(x = clip_9, y = var_589)[name = tensor("op_590")]; + tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_590)[name = tensor("clip_11")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_30, interleave = input_61_interleave_0, values = (window_11, window_13, window_15))[name = tensor("input_61")]; + tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; + tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; + tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_13_pad_type_0 = const()[name = tensor("inputs_13_pad_type_0"), val = tensor("valid")]; + tensor inputs_13_strides_0 = const()[name = tensor("inputs_13_strides_0"), val = tensor([1])]; + tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; + tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; + tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; + tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; + tensor var_613_split_sizes_0 = const()[name = tensor("op_613_split_sizes_0"), val = tensor([256, 256])]; + tensor var_613_axis_0 = const()[name = tensor("op_613_axis_0"), val = tensor(1)]; + tensor var_613_0, tensor var_613_1 = split(axis = var_613_axis_0, split_sizes = var_613_split_sizes_0, x = inputs_13)[name = tensor("op_613")]; + tensor var_615 = sigmoid(x = var_613_1)[name = tensor("op_615")]; + tensor inputs_15 = mul(x = var_613_0, y = var_615)[name = tensor("inputs_15")]; + tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; + tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; + tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; + tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_15)[name = tensor("outputs_aug_3")]; + tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; + tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([3, 256, 16])]; + tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; + tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_3)[name = tensor("input_65")]; + tensor inputs_17 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_65)[name = tensor("inputs_17")]; + tensor input_67 = silu(x = inputs_17)[name = tensor("input_67")]; + tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; + tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; + tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; + tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; + tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; + tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; + tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; + tensor var_646_begin_0 = const()[name = tensor("op_646_begin_0"), val = tensor([0, -1, 0])]; + tensor var_646_end_0 = const()[name = tensor("op_646_end_0"), val = tensor([3, 16, 256])]; + tensor var_646_end_mask_0 = const()[name = tensor("op_646_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_646 = slice_by_index(begin = var_646_begin_0, end = var_646_end_0, end_mask = var_646_end_mask_0, x = conv_out_3)[name = tensor("op_646")]; + tensor var_648_perm_0 = const()[name = tensor("op_648_perm_0"), val = tensor([1, 0, 2])]; + tensor var_648 = transpose(perm = var_648_perm_0, x = var_646)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_648)[name = tensor("input_71")]; + tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; + tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; + tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; + tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; + tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; + tensor var_671 = const()[name = tensor("op_671"), val = tensor(0x1p-1)]; + tensor var_672 = mul(x = input_79, y = var_671)[name = tensor("op_672")]; + tensor input_81 = add(x = var_672, y = input_71)[name = tensor("input_81")]; + tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; + tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; + tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; + tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; + tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; + tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; + tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; + tensor var_701 = const()[name = tensor("op_701"), val = tensor(0x1p-1)]; + tensor var_702 = mul(x = input_91, y = var_701)[name = tensor("op_702")]; + tensor input_93 = add(x = var_702, y = input_83)[name = tensor("input_93")]; + tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; + tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; tensor prev_kv_5_end_0 = const()[name = tensor("prev_kv_5_end_0"), val = tensor([3, 1, 4, 64, 64])]; tensor prev_kv_5_end_mask_0 = const()[name = tensor("prev_kv_5_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -883,316 +732,240 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_930 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_21)[name = tensor("linear_21")]; - tensor var_931 = const()[name = tensor("op_931"), val = tensor([1, 3, 4, 64])]; - tensor var_932 = reshape(shape = var_931, x = var_930)[name = tensor("op_932")]; + tensor var_716 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_717 = const()[name = tensor("op_717"), val = tensor([1, 3, 4, 64])]; + tensor var_718 = reshape(shape = var_717, x = var_716)[name = tensor("op_718")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_936 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_21)[name = tensor("linear_22")]; - tensor var_937 = const()[name = tensor("op_937"), val = tensor(0x1p-3)]; - tensor var_938 = mul(x = var_936, y = var_937)[name = tensor("op_938")]; - tensor var_939 = const()[name = tensor("op_939"), val = tensor([1, 3, 4, 64])]; - tensor var_940 = reshape(shape = var_939, x = var_938)[name = tensor("op_940")]; + tensor var_722 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_723 = const()[name = tensor("op_723"), val = tensor(0x1p-3)]; + tensor var_724 = mul(x = var_722, y = var_723)[name = tensor("op_724")]; + tensor var_725 = const()[name = tensor("op_725"), val = tensor([1, 3, 4, 64])]; + tensor var_726 = reshape(shape = var_725, x = var_724)[name = tensor("op_726")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_944 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_21)[name = tensor("linear_23")]; - tensor var_945 = const()[name = tensor("op_945"), val = tensor([1, 3, 4, 64])]; - tensor var_946 = reshape(shape = var_945, x = var_944)[name = tensor("op_946")]; + tensor var_730 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_731 = const()[name = tensor("op_731"), val = tensor([1, 3, 4, 64])]; + tensor var_732 = reshape(shape = var_731, x = var_730)[name = tensor("op_732")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_137 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_21)[name = tensor("linear_24")]; + tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; tensor s_t_5 = add(x = prev_scale_5, y = encoder__t_index)[name = tensor("s_t_5")]; tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_940)[name = tensor("transpose_49")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_932)[name = tensor("transpose_50")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_726)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_718)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_956 = const()[name = tensor("op_956"), val = tensor([3, 1])]; - tensor var_957 = reshape(shape = var_956, x = sqrt_s_t_5)[name = tensor("op_957")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_957)[name = tensor("M_5")]; - tensor var_959 = mul(x = qk_5, y = M_5)[name = tensor("op_959")]; + tensor var_742 = const()[name = tensor("op_742"), val = tensor([3, 1])]; + tensor var_743 = reshape(shape = var_742, x = sqrt_s_t_5)[name = tensor("op_743")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_743)[name = tensor("M_5")]; + tensor var_745 = mul(x = qk_5, y = M_5)[name = tensor("op_745")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_946)[name = tensor("transpose_48")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_959, y = v_5)[name = tensor("inner_5")]; - tensor var_961_transpose_x_0 = const()[name = tensor("op_961_transpose_x_0"), val = tensor(false)]; - tensor var_961_transpose_y_0 = const()[name = tensor("op_961_transpose_y_0"), val = tensor(false)]; - tensor var_961 = matmul(transpose_x = var_961_transpose_x_0, transpose_y = var_961_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_961")]; - tensor var_962 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_962")]; - tensor var_963 = const()[name = tensor("op_963"), val = tensor([1, 1, 3, 1])]; - tensor var_964 = reshape(shape = var_963, x = var_962)[name = tensor("op_964")]; - tensor cross_5 = mul(x = var_961, y = var_964)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_732)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_745, y = v_5)[name = tensor("inner_5")]; + tensor var_747_transpose_x_0 = const()[name = tensor("op_747_transpose_x_0"), val = tensor(false)]; + tensor var_747_transpose_y_0 = const()[name = tensor("op_747_transpose_y_0"), val = tensor(false)]; + tensor var_747 = matmul(transpose_x = var_747_transpose_x_0, transpose_y = var_747_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_747")]; + tensor var_748 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_748")]; + tensor var_749 = const()[name = tensor("op_749"), val = tensor([1, 1, 3, 1])]; + tensor var_750 = reshape(shape = var_749, x = var_748)[name = tensor("op_750")]; + tensor cross_5 = mul(x = var_747, y = var_750)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_967 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_967")]; - tensor var_969_transpose_x_1 = const()[name = tensor("op_969_transpose_x_1"), val = tensor(true)]; - tensor var_969_transpose_y_1 = const()[name = tensor("op_969_transpose_y_1"), val = tensor(false)]; - tensor var_969 = matmul(transpose_x = var_969_transpose_x_1, transpose_y = var_969_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_969")]; - tensor new_kv_unnorm_5 = add(x = var_967, y = var_969)[name = tensor("new_kv_unnorm_5")]; - tensor var_971 = const()[name = tensor("op_971"), val = tensor(0x1.8p+1)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_971)[name = tensor("new_scale_5")]; - tensor var_973 = sqrt(x = new_scale_5)[name = tensor("op_973")]; - tensor var_974 = real_div(x = new_kv_unnorm_5, y = var_973)[name = tensor("op_974")]; - tensor var_975_perm_0 = const()[name = tensor("op_975_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_753 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_753")]; + tensor var_755_transpose_x_1 = const()[name = tensor("op_755_transpose_x_1"), val = tensor(true)]; + tensor var_755_transpose_y_1 = const()[name = tensor("op_755_transpose_y_1"), val = tensor(false)]; + tensor var_755 = matmul(transpose_x = var_755_transpose_x_1, transpose_y = var_755_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_755")]; + tensor new_kv_unnorm_5 = add(x = var_753, y = var_755)[name = tensor("new_kv_unnorm_5")]; + tensor var_757 = const()[name = tensor("op_757"), val = tensor(0x1.8p+1)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_757)[name = tensor("new_scale_5")]; + tensor var_759 = sqrt(x = new_scale_5)[name = tensor("op_759")]; + tensor var_760 = real_div(x = new_kv_unnorm_5, y = var_759)[name = tensor("op_760")]; + tensor var_761_perm_0 = const()[name = tensor("op_761_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_975 = transpose(perm = var_975_perm_0, x = out_13)[name = tensor("transpose_47")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_975)[name = tensor("out_15")]; - tensor var_979 = const()[name = tensor("op_979"), val = tensor([1, 3, 256])]; - tensor out_17 = reshape(shape = var_979, x = out_15)[name = tensor("out_17")]; - tensor var_981 = silu(x = input_137)[name = tensor("op_981")]; - tensor input_139 = mul(x = var_981, y = out_17)[name = tensor("input_139")]; - tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_139)[name = tensor("linear_25")]; - tensor x_23 = add(x = input_133, y = ret_out_5)[name = tensor("x_23")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor var_761 = transpose(perm = var_761_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_761)[name = tensor("out_15")]; + tensor var_765 = const()[name = tensor("op_765"), val = tensor([1, 3, 256])]; + tensor out_17 = reshape(shape = var_765, x = out_15)[name = tensor("out_17")]; + tensor var_767 = silu(x = input_97)[name = tensor("op_767")]; + tensor input_99 = mul(x = var_767, y = out_17)[name = tensor("input_99")]; + tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; + tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; + tensor window_17_begin_0 = const()[name = tensor("window_17_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_17_end_0 = const()[name = tensor("window_17_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_17_end_mask_0 = const()[name = tensor("window_17_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_17_squeeze_mask_0 = const()[name = tensor("window_17_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_17 = slice_by_index(begin = window_17_begin_0, end = window_17_end_0, end_mask = window_17_end_mask_0, squeeze_mask = window_17_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_17")]; tensor filled_13_begin_0 = const()[name = tensor("filled_13_begin_0"), val = tensor([2, 0])]; tensor filled_13_end_0 = const()[name = tensor("filled_13_end_0"), val = tensor([3, 1])]; tensor filled_13_end_mask_0 = const()[name = tensor("filled_13_end_mask_0"), val = tensor([false, true])]; tensor filled_13_squeeze_mask_0 = const()[name = tensor("filled_13_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_13 = slice_by_index(begin = filled_13_begin_0, end = filled_13_end_0, end_mask = filled_13_end_mask_0, squeeze_mask = filled_13_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_13")]; - tensor var_990_begin_0 = const()[name = tensor("op_990_begin_0"), val = tensor([0, 0, 0])]; - tensor var_990_end_0 = const()[name = tensor("op_990_end_0"), val = tensor([1, 1, 256])]; - tensor var_990_end_mask_0 = const()[name = tensor("op_990_end_mask_0"), val = tensor([true, false, true])]; - tensor var_990 = slice_by_index(begin = var_990_begin_0, end = var_990_end_0, end_mask = var_990_end_mask_0, x = x_23)[name = tensor("op_990")]; - tensor var_993_begin_0 = const()[name = tensor("op_993_begin_0"), val = tensor([0, 1, 0])]; - tensor var_993_end_0 = const()[name = tensor("op_993_end_0"), val = tensor([1, 16, 256])]; - tensor var_993_end_mask_0 = const()[name = tensor("op_993_end_mask_0"), val = tensor([true, true, true])]; - tensor var_993 = slice_by_index(begin = var_993_begin_0, end = var_993_end_0, end_mask = var_993_end_mask_0, x = window_5)[name = tensor("op_993")]; + tensor var_776_begin_0 = const()[name = tensor("op_776_begin_0"), val = tensor([0, 0, 0])]; + tensor var_776_end_0 = const()[name = tensor("op_776_end_0"), val = tensor([1, 1, 256])]; + tensor var_776_end_mask_0 = const()[name = tensor("op_776_end_mask_0"), val = tensor([true, false, true])]; + tensor var_776 = slice_by_index(begin = var_776_begin_0, end = var_776_end_0, end_mask = var_776_end_mask_0, x = x_15)[name = tensor("op_776")]; + tensor var_779_begin_0 = const()[name = tensor("op_779_begin_0"), val = tensor([0, 1, 0])]; + tensor var_779_end_0 = const()[name = tensor("op_779_end_0"), val = tensor([1, 16, 256])]; + tensor var_779_end_mask_0 = const()[name = tensor("op_779_end_mask_0"), val = tensor([true, true, true])]; + tensor var_779 = slice_by_index(begin = var_779_begin_0, end = var_779_end_0, end_mask = var_779_end_mask_0, x = window_17)[name = tensor("op_779")]; tensor rolled_13_interleave_0 = const()[name = tensor("rolled_13_interleave_0"), val = tensor(false)]; - tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_993, var_990))[name = tensor("rolled_13")]; + tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_779, var_776))[name = tensor("rolled_13")]; tensor clip_12 = clip(alpha = var_21, beta = var_20, x = filled_13)[name = tensor("clip_12")]; - tensor var_998 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_998")]; - tensor var_999_dtype_0 = const()[name = tensor("op_999_dtype_0"), val = tensor("fp32")]; - tensor var_1000 = const()[name = tensor("op_1000"), val = tensor([1, 16, 1])]; - tensor var_999 = cast(dtype = var_999_dtype_0, x = var_998)[name = tensor("cast_147")]; - tensor fill_mask_13 = reshape(shape = var_1000, x = var_999)[name = tensor("fill_mask_13")]; + tensor var_784 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_784")]; + tensor var_785_dtype_0 = const()[name = tensor("op_785_dtype_0"), val = tensor("fp32")]; + tensor var_786 = const()[name = tensor("op_786"), val = tensor([1, 16, 1])]; + tensor var_785 = cast(dtype = var_785_dtype_0, x = var_784)[name = tensor("cast_147")]; + tensor fill_mask_13 = reshape(shape = var_786, x = var_785)[name = tensor("fill_mask_13")]; tensor x_expanded_13_reps_0 = const()[name = tensor("x_expanded_13_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_990)[name = tensor("x_expanded_13")]; - tensor var_1004 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_1004")]; - tensor var_1005 = mul(x = window_5, y = var_1004)[name = tensor("op_1005")]; - tensor var_1006 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_1006")]; - tensor filled_window_13 = add(x = var_1005, y = var_1006)[name = tensor("filled_window_13")]; - tensor var_1008 = greater_equal(x = filled_13, y = var_16)[name = tensor("op_1008")]; - tensor var_1009_dtype_0 = const()[name = tensor("op_1009_dtype_0"), val = tensor("fp32")]; - tensor var_1010 = const()[name = tensor("op_1010"), val = tensor([1, 1, 1])]; - tensor var_1009 = cast(dtype = var_1009_dtype_0, x = var_1008)[name = tensor("cast_146")]; - tensor use_roll_13 = reshape(shape = var_1010, x = var_1009)[name = tensor("use_roll_13")]; - tensor var_1012 = sub(x = var_17, y = use_roll_13)[name = tensor("op_1012")]; - tensor var_1013 = mul(x = filled_window_13, y = var_1012)[name = tensor("op_1013")]; - tensor var_1014 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_1014")]; - tensor input_141 = add(x = var_1013, y = var_1014)[name = tensor("input_141")]; - tensor var_1016 = const()[name = tensor("op_1016"), val = tensor(0x1p+0)]; - tensor var_1017 = add(x = filled_13, y = var_1016)[name = tensor("op_1017")]; - tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_1017)[name = tensor("clip_13")]; - tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; - tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_141)[name = tensor("x_25")]; - tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_47_pad_type_0 = const()[name = tensor("inputs_47_pad_type_0"), val = tensor("valid")]; - tensor inputs_47_strides_0 = const()[name = tensor("inputs_47_strides_0"), val = tensor([1])]; - tensor inputs_47_pad_0 = const()[name = tensor("inputs_47_pad_0"), val = tensor([0, 0])]; - tensor inputs_47_dilations_0 = const()[name = tensor("inputs_47_dilations_0"), val = tensor([1])]; - tensor inputs_47_groups_0 = const()[name = tensor("inputs_47_groups_0"), val = tensor(1)]; - tensor input_143 = transpose(perm = input_143_perm_0, x = x_25)[name = tensor("transpose_46")]; - tensor inputs_47 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_47_dilations_0, groups = inputs_47_groups_0, pad = inputs_47_pad_0, pad_type = inputs_47_pad_type_0, strides = inputs_47_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_47")]; - tensor var_1038_split_sizes_0 = const()[name = tensor("op_1038_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1038_axis_0 = const()[name = tensor("op_1038_axis_0"), val = tensor(1)]; - tensor var_1038_0, tensor var_1038_1 = split(axis = var_1038_axis_0, split_sizes = var_1038_split_sizes_0, x = inputs_47)[name = tensor("op_1038")]; - tensor var_1040 = sigmoid(x = var_1038_1)[name = tensor("op_1040")]; - tensor inputs_49 = mul(x = var_1038_0, y = var_1040)[name = tensor("inputs_49")]; - tensor outputs_aug_13_pad_type_0 = const()[name = tensor("outputs_aug_13_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_13_pad_0 = const()[name = tensor("outputs_aug_13_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_13_groups_0 = const()[name = tensor("outputs_aug_13_groups_0"), val = tensor(256)]; - tensor outputs_aug_13_strides_0 = const()[name = tensor("outputs_aug_13_strides_0"), val = tensor([1])]; - tensor outputs_aug_13_dilations_0 = const()[name = tensor("outputs_aug_13_dilations_0"), val = tensor([1])]; - tensor outputs_aug_13 = conv(dilations = outputs_aug_13_dilations_0, groups = outputs_aug_13_groups_0, pad = outputs_aug_13_pad_0, pad_type = outputs_aug_13_pad_type_0, strides = outputs_aug_13_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_49)[name = tensor("outputs_aug_13")]; - tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; - tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([1, 256, 16])]; - tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; - tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug_13)[name = tensor("input_145")]; - tensor inputs_51 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_145)[name = tensor("inputs_51")]; - tensor input_147 = silu(x = inputs_51)[name = tensor("input_147")]; - tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; - tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; - tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; - tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; - tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; - tensor input_149 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; - tensor conv_out_13_perm_0 = const()[name = tensor("conv_out_13_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1071_begin_0 = const()[name = tensor("op_1071_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1071_end_0 = const()[name = tensor("op_1071_end_0"), val = tensor([1, 16, 256])]; - tensor var_1071_end_mask_0 = const()[name = tensor("op_1071_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_13 = transpose(perm = conv_out_13_perm_0, x = input_149)[name = tensor("transpose_45")]; - tensor var_1071 = slice_by_index(begin = var_1071_begin_0, end = var_1071_end_0, end_mask = var_1071_end_mask_0, x = conv_out_13)[name = tensor("op_1071")]; - tensor var_1074_begin_0 = const()[name = tensor("op_1074_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1074_end_0 = const()[name = tensor("op_1074_end_0"), val = tensor([1, 2, 256])]; - tensor var_1074_end_mask_0 = const()[name = tensor("op_1074_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1074 = slice_by_index(begin = var_1074_begin_0, end = var_1074_end_0, end_mask = var_1074_end_mask_0, x = x_23)[name = tensor("op_1074")]; - tensor var_1077_begin_0 = const()[name = tensor("op_1077_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1077_end_0 = const()[name = tensor("op_1077_end_0"), val = tensor([1, 16, 256])]; - tensor var_1077_end_mask_0 = const()[name = tensor("op_1077_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1077 = slice_by_index(begin = var_1077_begin_0, end = var_1077_end_0, end_mask = var_1077_end_mask_0, x = input_141)[name = tensor("op_1077")]; + tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_776)[name = tensor("x_expanded_13")]; + tensor var_790 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_790")]; + tensor var_791 = mul(x = window_17, y = var_790)[name = tensor("op_791")]; + tensor var_792 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_792")]; + tensor filled_window_13 = add(x = var_791, y = var_792)[name = tensor("filled_window_13")]; + tensor var_794 = greater_equal(x = filled_13, y = var_16)[name = tensor("op_794")]; + tensor var_795_dtype_0 = const()[name = tensor("op_795_dtype_0"), val = tensor("fp32")]; + tensor var_796 = const()[name = tensor("op_796"), val = tensor([1, 1, 1])]; + tensor var_795 = cast(dtype = var_795_dtype_0, x = var_794)[name = tensor("cast_146")]; + tensor use_roll_13 = reshape(shape = var_796, x = var_795)[name = tensor("use_roll_13")]; + tensor var_798 = sub(x = var_17, y = use_roll_13)[name = tensor("op_798")]; + tensor var_799 = mul(x = filled_window_13, y = var_798)[name = tensor("op_799")]; + tensor var_800 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_800")]; + tensor window_19 = add(x = var_799, y = var_800)[name = tensor("window_19")]; + tensor var_802 = const()[name = tensor("op_802"), val = tensor(0x1p+0)]; + tensor var_803 = add(x = filled_13, y = var_802)[name = tensor("op_803")]; + tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_803)[name = tensor("clip_13")]; + tensor var_806_begin_0 = const()[name = tensor("op_806_begin_0"), val = tensor([0, 1, 0])]; + tensor var_806_end_0 = const()[name = tensor("op_806_end_0"), val = tensor([1, 2, 256])]; + tensor var_806_end_mask_0 = const()[name = tensor("op_806_end_mask_0"), val = tensor([true, false, true])]; + tensor var_806 = slice_by_index(begin = var_806_begin_0, end = var_806_end_0, end_mask = var_806_end_mask_0, x = x_15)[name = tensor("op_806")]; + tensor var_809_begin_0 = const()[name = tensor("op_809_begin_0"), val = tensor([0, 1, 0])]; + tensor var_809_end_0 = const()[name = tensor("op_809_end_0"), val = tensor([1, 16, 256])]; + tensor var_809_end_mask_0 = const()[name = tensor("op_809_end_mask_0"), val = tensor([true, true, true])]; + tensor var_809 = slice_by_index(begin = var_809_begin_0, end = var_809_end_0, end_mask = var_809_end_mask_0, x = window_19)[name = tensor("op_809")]; tensor rolled_15_interleave_0 = const()[name = tensor("rolled_15_interleave_0"), val = tensor(false)]; - tensor rolled_15 = concat(axis = var_33, interleave = rolled_15_interleave_0, values = (var_1077, var_1074))[name = tensor("rolled_15")]; + tensor rolled_15 = concat(axis = var_33, interleave = rolled_15_interleave_0, values = (var_809, var_806))[name = tensor("rolled_15")]; tensor clip_14 = clip(alpha = var_21, beta = var_20, x = clip_13)[name = tensor("clip_14")]; - tensor var_1082 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_1082")]; - tensor var_1083_dtype_0 = const()[name = tensor("op_1083_dtype_0"), val = tensor("fp32")]; - tensor var_1084 = const()[name = tensor("op_1084"), val = tensor([1, 16, 1])]; - tensor var_1083 = cast(dtype = var_1083_dtype_0, x = var_1082)[name = tensor("cast_145")]; - tensor fill_mask_15 = reshape(shape = var_1084, x = var_1083)[name = tensor("fill_mask_15")]; + tensor var_814 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_814")]; + tensor var_815_dtype_0 = const()[name = tensor("op_815_dtype_0"), val = tensor("fp32")]; + tensor var_816 = const()[name = tensor("op_816"), val = tensor([1, 16, 1])]; + tensor var_815 = cast(dtype = var_815_dtype_0, x = var_814)[name = tensor("cast_145")]; + tensor fill_mask_15 = reshape(shape = var_816, x = var_815)[name = tensor("fill_mask_15")]; tensor x_expanded_15_reps_0 = const()[name = tensor("x_expanded_15_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_1074)[name = tensor("x_expanded_15")]; - tensor var_1088 = sub(x = var_17, y = fill_mask_15)[name = tensor("op_1088")]; - tensor var_1089 = mul(x = input_141, y = var_1088)[name = tensor("op_1089")]; - tensor var_1090 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_1090")]; - tensor filled_window_15 = add(x = var_1089, y = var_1090)[name = tensor("filled_window_15")]; - tensor var_1092 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_1092")]; - tensor var_1093_dtype_0 = const()[name = tensor("op_1093_dtype_0"), val = tensor("fp32")]; - tensor var_1094 = const()[name = tensor("op_1094"), val = tensor([1, 1, 1])]; - tensor var_1093 = cast(dtype = var_1093_dtype_0, x = var_1092)[name = tensor("cast_144")]; - tensor use_roll_15 = reshape(shape = var_1094, x = var_1093)[name = tensor("use_roll_15")]; - tensor var_1096 = sub(x = var_17, y = use_roll_15)[name = tensor("op_1096")]; - tensor var_1097 = mul(x = filled_window_15, y = var_1096)[name = tensor("op_1097")]; - tensor var_1098 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_1098")]; - tensor input_151 = add(x = var_1097, y = var_1098)[name = tensor("input_151")]; - tensor var_1100 = const()[name = tensor("op_1100"), val = tensor(0x1p+0)]; - tensor var_1101 = add(x = clip_13, y = var_1100)[name = tensor("op_1101")]; - tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_1101)[name = tensor("clip_15")]; - tensor x_27_axes_0 = const()[name = tensor("x_27_axes_0"), val = tensor([-1])]; - tensor x_27 = layer_norm(axes = x_27_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_151)[name = tensor("x_27")]; - tensor input_153_perm_0 = const()[name = tensor("input_153_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_53_pad_type_0 = const()[name = tensor("inputs_53_pad_type_0"), val = tensor("valid")]; - tensor inputs_53_strides_0 = const()[name = tensor("inputs_53_strides_0"), val = tensor([1])]; - tensor inputs_53_pad_0 = const()[name = tensor("inputs_53_pad_0"), val = tensor([0, 0])]; - tensor inputs_53_dilations_0 = const()[name = tensor("inputs_53_dilations_0"), val = tensor([1])]; - tensor inputs_53_groups_0 = const()[name = tensor("inputs_53_groups_0"), val = tensor(1)]; - tensor input_153 = transpose(perm = input_153_perm_0, x = x_27)[name = tensor("transpose_44")]; - tensor inputs_53 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_53_dilations_0, groups = inputs_53_groups_0, pad = inputs_53_pad_0, pad_type = inputs_53_pad_type_0, strides = inputs_53_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_153)[name = tensor("inputs_53")]; - tensor var_1122_split_sizes_0 = const()[name = tensor("op_1122_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1122_axis_0 = const()[name = tensor("op_1122_axis_0"), val = tensor(1)]; - tensor var_1122_0, tensor var_1122_1 = split(axis = var_1122_axis_0, split_sizes = var_1122_split_sizes_0, x = inputs_53)[name = tensor("op_1122")]; - tensor var_1124 = sigmoid(x = var_1122_1)[name = tensor("op_1124")]; - tensor inputs_55 = mul(x = var_1122_0, y = var_1124)[name = tensor("inputs_55")]; - tensor outputs_aug_15_pad_type_0 = const()[name = tensor("outputs_aug_15_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_15_pad_0 = const()[name = tensor("outputs_aug_15_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_15_groups_0 = const()[name = tensor("outputs_aug_15_groups_0"), val = tensor(256)]; - tensor outputs_aug_15_strides_0 = const()[name = tensor("outputs_aug_15_strides_0"), val = tensor([1])]; - tensor outputs_aug_15_dilations_0 = const()[name = tensor("outputs_aug_15_dilations_0"), val = tensor([1])]; - tensor outputs_aug_15 = conv(dilations = outputs_aug_15_dilations_0, groups = outputs_aug_15_groups_0, pad = outputs_aug_15_pad_0, pad_type = outputs_aug_15_pad_type_0, strides = outputs_aug_15_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_55)[name = tensor("outputs_aug_15")]; - tensor input_155_begin_0 = const()[name = tensor("input_155_begin_0"), val = tensor([0, 0, 0])]; - tensor input_155_end_0 = const()[name = tensor("input_155_end_0"), val = tensor([1, 256, 16])]; - tensor input_155_end_mask_0 = const()[name = tensor("input_155_end_mask_0"), val = tensor([true, true, false])]; - tensor input_155 = slice_by_index(begin = input_155_begin_0, end = input_155_end_0, end_mask = input_155_end_mask_0, x = outputs_aug_15)[name = tensor("input_155")]; - tensor inputs_57 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_155)[name = tensor("inputs_57")]; - tensor input_157 = silu(x = inputs_57)[name = tensor("input_157")]; - tensor input_159_pad_type_0 = const()[name = tensor("input_159_pad_type_0"), val = tensor("valid")]; - tensor input_159_strides_0 = const()[name = tensor("input_159_strides_0"), val = tensor([1])]; - tensor input_159_pad_0 = const()[name = tensor("input_159_pad_0"), val = tensor([0, 0])]; - tensor input_159_dilations_0 = const()[name = tensor("input_159_dilations_0"), val = tensor([1])]; - tensor input_159_groups_0 = const()[name = tensor("input_159_groups_0"), val = tensor(1)]; - tensor input_159 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_159_dilations_0, groups = input_159_groups_0, pad = input_159_pad_0, pad_type = input_159_pad_type_0, strides = input_159_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_157)[name = tensor("input_159")]; - tensor conv_out_15_perm_0 = const()[name = tensor("conv_out_15_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1155_begin_0 = const()[name = tensor("op_1155_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1155_end_0 = const()[name = tensor("op_1155_end_0"), val = tensor([1, 16, 256])]; - tensor var_1155_end_mask_0 = const()[name = tensor("op_1155_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_15 = transpose(perm = conv_out_15_perm_0, x = input_159)[name = tensor("transpose_43")]; - tensor var_1155 = slice_by_index(begin = var_1155_begin_0, end = var_1155_end_0, end_mask = var_1155_end_mask_0, x = conv_out_15)[name = tensor("op_1155")]; - tensor var_1158_begin_0 = const()[name = tensor("op_1158_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1158_end_0 = const()[name = tensor("op_1158_end_0"), val = tensor([1, 1, 256])]; - tensor var_1158_end_mask_0 = const()[name = tensor("op_1158_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1158 = slice_by_index(begin = var_1158_begin_0, end = var_1158_end_0, end_mask = var_1158_end_mask_0, x = x_23)[name = tensor("op_1158")]; - tensor var_1161_begin_0 = const()[name = tensor("op_1161_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1161_end_0 = const()[name = tensor("op_1161_end_0"), val = tensor([1, 16, 256])]; - tensor var_1161_end_mask_0 = const()[name = tensor("op_1161_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1161 = slice_by_index(begin = var_1161_begin_0, end = var_1161_end_0, end_mask = var_1161_end_mask_0, x = input_151)[name = tensor("op_1161")]; + tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_806)[name = tensor("x_expanded_15")]; + tensor var_820 = sub(x = var_17, y = fill_mask_15)[name = tensor("op_820")]; + tensor var_821 = mul(x = window_19, y = var_820)[name = tensor("op_821")]; + tensor var_822 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_822")]; + tensor filled_window_15 = add(x = var_821, y = var_822)[name = tensor("filled_window_15")]; + tensor var_824 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_824")]; + tensor var_825_dtype_0 = const()[name = tensor("op_825_dtype_0"), val = tensor("fp32")]; + tensor var_826 = const()[name = tensor("op_826"), val = tensor([1, 1, 1])]; + tensor var_825 = cast(dtype = var_825_dtype_0, x = var_824)[name = tensor("cast_144")]; + tensor use_roll_15 = reshape(shape = var_826, x = var_825)[name = tensor("use_roll_15")]; + tensor var_828 = sub(x = var_17, y = use_roll_15)[name = tensor("op_828")]; + tensor var_829 = mul(x = filled_window_15, y = var_828)[name = tensor("op_829")]; + tensor var_830 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_830")]; + tensor window_21 = add(x = var_829, y = var_830)[name = tensor("window_21")]; + tensor var_832 = const()[name = tensor("op_832"), val = tensor(0x1p+0)]; + tensor var_833 = add(x = clip_13, y = var_832)[name = tensor("op_833")]; + tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_833)[name = tensor("clip_15")]; + tensor var_836_begin_0 = const()[name = tensor("op_836_begin_0"), val = tensor([0, 2, 0])]; + tensor var_836_end_0 = const()[name = tensor("op_836_end_0"), val = tensor([1, 1, 256])]; + tensor var_836_end_mask_0 = const()[name = tensor("op_836_end_mask_0"), val = tensor([true, true, true])]; + tensor var_836 = slice_by_index(begin = var_836_begin_0, end = var_836_end_0, end_mask = var_836_end_mask_0, x = x_15)[name = tensor("op_836")]; + tensor var_839_begin_0 = const()[name = tensor("op_839_begin_0"), val = tensor([0, 1, 0])]; + tensor var_839_end_0 = const()[name = tensor("op_839_end_0"), val = tensor([1, 16, 256])]; + tensor var_839_end_mask_0 = const()[name = tensor("op_839_end_mask_0"), val = tensor([true, true, true])]; + tensor var_839 = slice_by_index(begin = var_839_begin_0, end = var_839_end_0, end_mask = var_839_end_mask_0, x = window_21)[name = tensor("op_839")]; tensor rolled_17_interleave_0 = const()[name = tensor("rolled_17_interleave_0"), val = tensor(false)]; - tensor rolled_17 = concat(axis = var_33, interleave = rolled_17_interleave_0, values = (var_1161, var_1158))[name = tensor("rolled_17")]; + tensor rolled_17 = concat(axis = var_33, interleave = rolled_17_interleave_0, values = (var_839, var_836))[name = tensor("rolled_17")]; tensor clip_16 = clip(alpha = var_21, beta = var_20, x = clip_15)[name = tensor("clip_16")]; - tensor var_1166 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_1166")]; - tensor var_1167_dtype_0 = const()[name = tensor("op_1167_dtype_0"), val = tensor("fp32")]; - tensor var_1168 = const()[name = tensor("op_1168"), val = tensor([1, 16, 1])]; - tensor var_1167 = cast(dtype = var_1167_dtype_0, x = var_1166)[name = tensor("cast_143")]; - tensor fill_mask_17 = reshape(shape = var_1168, x = var_1167)[name = tensor("fill_mask_17")]; + tensor var_844 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_844")]; + tensor var_845_dtype_0 = const()[name = tensor("op_845_dtype_0"), val = tensor("fp32")]; + tensor var_846 = const()[name = tensor("op_846"), val = tensor([1, 16, 1])]; + tensor var_845 = cast(dtype = var_845_dtype_0, x = var_844)[name = tensor("cast_143")]; + tensor fill_mask_17 = reshape(shape = var_846, x = var_845)[name = tensor("fill_mask_17")]; tensor x_expanded_17_reps_0 = const()[name = tensor("x_expanded_17_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_1158)[name = tensor("x_expanded_17")]; - tensor var_1172 = sub(x = var_17, y = fill_mask_17)[name = tensor("op_1172")]; - tensor var_1173 = mul(x = input_151, y = var_1172)[name = tensor("op_1173")]; - tensor var_1174 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_1174")]; - tensor filled_window_17 = add(x = var_1173, y = var_1174)[name = tensor("filled_window_17")]; - tensor var_1176 = greater_equal(x = clip_15, y = var_16)[name = tensor("op_1176")]; - tensor var_1177_dtype_0 = const()[name = tensor("op_1177_dtype_0"), val = tensor("fp32")]; - tensor var_1178 = const()[name = tensor("op_1178"), val = tensor([1, 1, 1])]; - tensor var_1177 = cast(dtype = var_1177_dtype_0, x = var_1176)[name = tensor("cast_142")]; - tensor use_roll_17 = reshape(shape = var_1178, x = var_1177)[name = tensor("use_roll_17")]; - tensor var_1180 = sub(x = var_17, y = use_roll_17)[name = tensor("op_1180")]; - tensor var_1181 = mul(x = filled_window_17, y = var_1180)[name = tensor("op_1181")]; - tensor var_1182 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_1182")]; - tensor input_161 = add(x = var_1181, y = var_1182)[name = tensor("input_161")]; - tensor var_1184 = const()[name = tensor("op_1184"), val = tensor(0x1p+0)]; - tensor var_1185 = add(x = clip_15, y = var_1184)[name = tensor("op_1185")]; - tensor clip_17 = clip(alpha = var_21, beta = var_16, x = var_1185)[name = tensor("clip_17")]; - tensor x_29_axes_0 = const()[name = tensor("x_29_axes_0"), val = tensor([-1])]; - tensor x_29 = layer_norm(axes = x_29_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_161)[name = tensor("x_29")]; - tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_59_pad_type_0 = const()[name = tensor("inputs_59_pad_type_0"), val = tensor("valid")]; - tensor inputs_59_strides_0 = const()[name = tensor("inputs_59_strides_0"), val = tensor([1])]; - tensor inputs_59_pad_0 = const()[name = tensor("inputs_59_pad_0"), val = tensor([0, 0])]; - tensor inputs_59_dilations_0 = const()[name = tensor("inputs_59_dilations_0"), val = tensor([1])]; - tensor inputs_59_groups_0 = const()[name = tensor("inputs_59_groups_0"), val = tensor(1)]; - tensor input_163 = transpose(perm = input_163_perm_0, x = x_29)[name = tensor("transpose_42")]; - tensor inputs_59 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_59_dilations_0, groups = inputs_59_groups_0, pad = inputs_59_pad_0, pad_type = inputs_59_pad_type_0, strides = inputs_59_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_163)[name = tensor("inputs_59")]; - tensor var_1206_split_sizes_0 = const()[name = tensor("op_1206_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1206_axis_0 = const()[name = tensor("op_1206_axis_0"), val = tensor(1)]; - tensor var_1206_0, tensor var_1206_1 = split(axis = var_1206_axis_0, split_sizes = var_1206_split_sizes_0, x = inputs_59)[name = tensor("op_1206")]; - tensor var_1208 = sigmoid(x = var_1206_1)[name = tensor("op_1208")]; - tensor inputs_61 = mul(x = var_1206_0, y = var_1208)[name = tensor("inputs_61")]; - tensor outputs_aug_17_pad_type_0 = const()[name = tensor("outputs_aug_17_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_17_pad_0 = const()[name = tensor("outputs_aug_17_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_17_groups_0 = const()[name = tensor("outputs_aug_17_groups_0"), val = tensor(256)]; - tensor outputs_aug_17_strides_0 = const()[name = tensor("outputs_aug_17_strides_0"), val = tensor([1])]; - tensor outputs_aug_17_dilations_0 = const()[name = tensor("outputs_aug_17_dilations_0"), val = tensor([1])]; - tensor outputs_aug_17 = conv(dilations = outputs_aug_17_dilations_0, groups = outputs_aug_17_groups_0, pad = outputs_aug_17_pad_0, pad_type = outputs_aug_17_pad_type_0, strides = outputs_aug_17_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_61)[name = tensor("outputs_aug_17")]; - tensor input_165_begin_0 = const()[name = tensor("input_165_begin_0"), val = tensor([0, 0, 0])]; - tensor input_165_end_0 = const()[name = tensor("input_165_end_0"), val = tensor([1, 256, 16])]; - tensor input_165_end_mask_0 = const()[name = tensor("input_165_end_mask_0"), val = tensor([true, true, false])]; - tensor input_165 = slice_by_index(begin = input_165_begin_0, end = input_165_end_0, end_mask = input_165_end_mask_0, x = outputs_aug_17)[name = tensor("input_165")]; - tensor inputs_63 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_165)[name = tensor("inputs_63")]; - tensor input_167 = silu(x = inputs_63)[name = tensor("input_167")]; - tensor input_169_pad_type_0 = const()[name = tensor("input_169_pad_type_0"), val = tensor("valid")]; - tensor input_169_strides_0 = const()[name = tensor("input_169_strides_0"), val = tensor([1])]; - tensor input_169_pad_0 = const()[name = tensor("input_169_pad_0"), val = tensor([0, 0])]; - tensor input_169_dilations_0 = const()[name = tensor("input_169_dilations_0"), val = tensor([1])]; - tensor input_169_groups_0 = const()[name = tensor("input_169_groups_0"), val = tensor(1)]; - tensor input_169 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_169_dilations_0, groups = input_169_groups_0, pad = input_169_pad_0, pad_type = input_169_pad_type_0, strides = input_169_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_167)[name = tensor("input_169")]; - tensor conv_out_17_perm_0 = const()[name = tensor("conv_out_17_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1239_begin_0 = const()[name = tensor("op_1239_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1239_end_0 = const()[name = tensor("op_1239_end_0"), val = tensor([1, 16, 256])]; - tensor var_1239_end_mask_0 = const()[name = tensor("op_1239_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_17 = transpose(perm = conv_out_17_perm_0, x = input_169)[name = tensor("transpose_41")]; - tensor var_1239 = slice_by_index(begin = var_1239_begin_0, end = var_1239_end_0, end_mask = var_1239_end_mask_0, x = conv_out_17)[name = tensor("op_1239")]; - tensor var_1242_interleave_0 = const()[name = tensor("op_1242_interleave_0"), val = tensor(false)]; - tensor var_1242 = concat(axis = var_33, interleave = var_1242_interleave_0, values = (var_1071, var_1155, var_1239))[name = tensor("op_1242")]; - tensor input_171 = add(x = x_23, y = var_1242)[name = tensor("input_171")]; - tensor input_173_axes_0 = const()[name = tensor("input_173_axes_0"), val = tensor([-1])]; - tensor input_173 = layer_norm(axes = input_173_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_171)[name = tensor("input_173")]; - tensor inputs_65 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_173)[name = tensor("linear_26")]; - tensor input_175 = silu(x = inputs_65)[name = tensor("input_175")]; - tensor input_179 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_175)[name = tensor("linear_27")]; - tensor var_1265 = const()[name = tensor("op_1265"), val = tensor(0x1p-1)]; - tensor var_1266 = mul(x = input_179, y = var_1265)[name = tensor("op_1266")]; - tensor input_181 = add(x = var_1266, y = input_171)[name = tensor("input_181")]; - tensor input_183_axes_0 = const()[name = tensor("input_183_axes_0"), val = tensor([-1])]; - tensor input_183 = layer_norm(axes = input_183_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_181)[name = tensor("input_183")]; - tensor input_185_axes_0 = const()[name = tensor("input_185_axes_0"), val = tensor([-1])]; - tensor input_185 = layer_norm(axes = input_185_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_183)[name = tensor("input_185")]; - tensor inputs_67 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_185)[name = tensor("linear_28")]; - tensor input_187 = silu(x = inputs_67)[name = tensor("input_187")]; - tensor input_191 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_187)[name = tensor("linear_29")]; - tensor var_1295 = const()[name = tensor("op_1295"), val = tensor(0x1p-1)]; - tensor var_1296 = mul(x = input_191, y = var_1295)[name = tensor("op_1296")]; - tensor input_193 = add(x = var_1296, y = input_183)[name = tensor("input_193")]; - tensor x_31_axes_0 = const()[name = tensor("x_31_axes_0"), val = tensor([-1])]; - tensor x_31 = layer_norm(axes = x_31_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_193)[name = tensor("x_31")]; + tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_836)[name = tensor("x_expanded_17")]; + tensor var_850 = sub(x = var_17, y = fill_mask_17)[name = tensor("op_850")]; + tensor var_851 = mul(x = window_21, y = var_850)[name = tensor("op_851")]; + tensor var_852 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_852")]; + tensor filled_window_17 = add(x = var_851, y = var_852)[name = tensor("filled_window_17")]; + tensor var_854 = greater_equal(x = clip_15, y = var_16)[name = tensor("op_854")]; + tensor var_855_dtype_0 = const()[name = tensor("op_855_dtype_0"), val = tensor("fp32")]; + tensor var_856 = const()[name = tensor("op_856"), val = tensor([1, 1, 1])]; + tensor var_855 = cast(dtype = var_855_dtype_0, x = var_854)[name = tensor("cast_142")]; + tensor use_roll_17 = reshape(shape = var_856, x = var_855)[name = tensor("use_roll_17")]; + tensor var_858 = sub(x = var_17, y = use_roll_17)[name = tensor("op_858")]; + tensor var_859 = mul(x = filled_window_17, y = var_858)[name = tensor("op_859")]; + tensor var_860 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_860")]; + tensor window_23 = add(x = var_859, y = var_860)[name = tensor("window_23")]; + tensor var_862 = const()[name = tensor("op_862"), val = tensor(0x1p+0)]; + tensor var_863 = add(x = clip_15, y = var_862)[name = tensor("op_863")]; + tensor clip_17 = clip(alpha = var_21, beta = var_16, x = var_863)[name = tensor("clip_17")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_30, interleave = input_101_interleave_0, values = (window_19, window_21, window_23))[name = tensor("input_101")]; + tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; + tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; + tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_23_pad_type_0 = const()[name = tensor("inputs_23_pad_type_0"), val = tensor("valid")]; + tensor inputs_23_strides_0 = const()[name = tensor("inputs_23_strides_0"), val = tensor([1])]; + tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; + tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; + tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; + tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; + tensor var_886_split_sizes_0 = const()[name = tensor("op_886_split_sizes_0"), val = tensor([256, 256])]; + tensor var_886_axis_0 = const()[name = tensor("op_886_axis_0"), val = tensor(1)]; + tensor var_886_0, tensor var_886_1 = split(axis = var_886_axis_0, split_sizes = var_886_split_sizes_0, x = inputs_23)[name = tensor("op_886")]; + tensor var_888 = sigmoid(x = var_886_1)[name = tensor("op_888")]; + tensor inputs_25 = mul(x = var_886_0, y = var_888)[name = tensor("inputs_25")]; + tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; + tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; + tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; + tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_25)[name = tensor("outputs_aug_5")]; + tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; + tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([3, 256, 16])]; + tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; + tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_5)[name = tensor("input_105")]; + tensor inputs_27 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_105)[name = tensor("inputs_27")]; + tensor input_107 = silu(x = inputs_27)[name = tensor("input_107")]; + tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; + tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; + tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; + tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; + tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; + tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; + tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; + tensor var_919_begin_0 = const()[name = tensor("op_919_begin_0"), val = tensor([0, -1, 0])]; + tensor var_919_end_0 = const()[name = tensor("op_919_end_0"), val = tensor([3, 16, 256])]; + tensor var_919_end_mask_0 = const()[name = tensor("op_919_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_919 = slice_by_index(begin = var_919_begin_0, end = var_919_end_0, end_mask = var_919_end_mask_0, x = conv_out_5)[name = tensor("op_919")]; + tensor var_921_perm_0 = const()[name = tensor("op_921_perm_0"), val = tensor([1, 0, 2])]; + tensor var_921 = transpose(perm = var_921_perm_0, x = var_919)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_921)[name = tensor("input_111")]; + tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; + tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; + tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; + tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; + tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; + tensor var_944 = const()[name = tensor("op_944"), val = tensor(0x1p-1)]; + tensor var_945 = mul(x = input_119, y = var_944)[name = tensor("op_945")]; + tensor input_121 = add(x = var_945, y = input_111)[name = tensor("input_121")]; + tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; + tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; + tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; + tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; + tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; + tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; + tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; + tensor var_974 = const()[name = tensor("op_974"), val = tensor(0x1p-1)]; + tensor var_975 = mul(x = input_131, y = var_974)[name = tensor("op_975")]; + tensor input_133 = add(x = var_975, y = input_123)[name = tensor("input_133")]; + tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; + tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; tensor prev_kv_7_end_0 = const()[name = tensor("prev_kv_7_end_0"), val = tensor([4, 1, 4, 64, 64])]; tensor prev_kv_7_end_mask_0 = const()[name = tensor("prev_kv_7_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1203,309 +976,233 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_1310 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_31)[name = tensor("linear_30")]; - tensor var_1311 = const()[name = tensor("op_1311"), val = tensor([1, 3, 4, 64])]; - tensor var_1312 = reshape(shape = var_1311, x = var_1310)[name = tensor("op_1312")]; + tensor var_989 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_990 = const()[name = tensor("op_990"), val = tensor([1, 3, 4, 64])]; + tensor var_991 = reshape(shape = var_990, x = var_989)[name = tensor("op_991")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1316 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_31)[name = tensor("linear_31")]; - tensor var_1317 = const()[name = tensor("op_1317"), val = tensor(0x1p-3)]; - tensor var_1318 = mul(x = var_1316, y = var_1317)[name = tensor("op_1318")]; - tensor var_1319 = const()[name = tensor("op_1319"), val = tensor([1, 3, 4, 64])]; - tensor var_1320 = reshape(shape = var_1319, x = var_1318)[name = tensor("op_1320")]; + tensor var_995 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_996 = const()[name = tensor("op_996"), val = tensor(0x1p-3)]; + tensor var_997 = mul(x = var_995, y = var_996)[name = tensor("op_997")]; + tensor var_998 = const()[name = tensor("op_998"), val = tensor([1, 3, 4, 64])]; + tensor var_999 = reshape(shape = var_998, x = var_997)[name = tensor("op_999")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1324 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_31)[name = tensor("linear_32")]; - tensor var_1325 = const()[name = tensor("op_1325"), val = tensor([1, 3, 4, 64])]; - tensor var_1326 = reshape(shape = var_1325, x = var_1324)[name = tensor("op_1326")]; + tensor var_1003 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_1004 = const()[name = tensor("op_1004"), val = tensor([1, 3, 4, 64])]; + tensor var_1005 = reshape(shape = var_1004, x = var_1003)[name = tensor("op_1005")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_197 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_31)[name = tensor("linear_33")]; + tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; tensor s_t_7 = add(x = prev_scale_7, y = encoder__t_index)[name = tensor("s_t_7")]; tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_1320)[name = tensor("transpose_39")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_1312)[name = tensor("transpose_40")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_999)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_991)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_1336 = const()[name = tensor("op_1336"), val = tensor([3, 1])]; - tensor var_1337 = reshape(shape = var_1336, x = sqrt_s_t_7)[name = tensor("op_1337")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_1337)[name = tensor("M_7")]; - tensor var_1339 = mul(x = qk_7, y = M_7)[name = tensor("op_1339")]; + tensor var_1015 = const()[name = tensor("op_1015"), val = tensor([3, 1])]; + tensor var_1016 = reshape(shape = var_1015, x = sqrt_s_t_7)[name = tensor("op_1016")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_1016)[name = tensor("M_7")]; + tensor var_1018 = mul(x = qk_7, y = M_7)[name = tensor("op_1018")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_1326)[name = tensor("transpose_38")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1339, y = v_7)[name = tensor("inner_7")]; - tensor var_1341_transpose_x_0 = const()[name = tensor("op_1341_transpose_x_0"), val = tensor(false)]; - tensor var_1341_transpose_y_0 = const()[name = tensor("op_1341_transpose_y_0"), val = tensor(false)]; - tensor var_1341 = matmul(transpose_x = var_1341_transpose_x_0, transpose_y = var_1341_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1341")]; - tensor var_1342 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1342")]; - tensor var_1343 = const()[name = tensor("op_1343"), val = tensor([1, 1, 3, 1])]; - tensor var_1344 = reshape(shape = var_1343, x = var_1342)[name = tensor("op_1344")]; - tensor cross_7 = mul(x = var_1341, y = var_1344)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_1005)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1018, y = v_7)[name = tensor("inner_7")]; + tensor var_1020_transpose_x_0 = const()[name = tensor("op_1020_transpose_x_0"), val = tensor(false)]; + tensor var_1020_transpose_y_0 = const()[name = tensor("op_1020_transpose_y_0"), val = tensor(false)]; + tensor var_1020 = matmul(transpose_x = var_1020_transpose_x_0, transpose_y = var_1020_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1020")]; + tensor var_1021 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1021")]; + tensor var_1022 = const()[name = tensor("op_1022"), val = tensor([1, 1, 3, 1])]; + tensor var_1023 = reshape(shape = var_1022, x = var_1021)[name = tensor("op_1023")]; + tensor cross_7 = mul(x = var_1020, y = var_1023)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_1347 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1347")]; - tensor var_1349_transpose_x_1 = const()[name = tensor("op_1349_transpose_x_1"), val = tensor(true)]; - tensor var_1349_transpose_y_1 = const()[name = tensor("op_1349_transpose_y_1"), val = tensor(false)]; - tensor var_1349 = matmul(transpose_x = var_1349_transpose_x_1, transpose_y = var_1349_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1349")]; - tensor new_kv_unnorm_7 = add(x = var_1347, y = var_1349)[name = tensor("new_kv_unnorm_7")]; - tensor var_1351 = const()[name = tensor("op_1351"), val = tensor(0x1.8p+1)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_1351)[name = tensor("new_scale_7")]; - tensor var_1353 = sqrt(x = new_scale_7)[name = tensor("op_1353")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1353)[name = tensor("nkv_1")]; - tensor var_1355_perm_0 = const()[name = tensor("op_1355_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1026 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1026")]; + tensor var_1028_transpose_x_1 = const()[name = tensor("op_1028_transpose_x_1"), val = tensor(true)]; + tensor var_1028_transpose_y_1 = const()[name = tensor("op_1028_transpose_y_1"), val = tensor(false)]; + tensor var_1028 = matmul(transpose_x = var_1028_transpose_x_1, transpose_y = var_1028_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1028")]; + tensor new_kv_unnorm_7 = add(x = var_1026, y = var_1028)[name = tensor("new_kv_unnorm_7")]; + tensor var_1030 = const()[name = tensor("op_1030"), val = tensor(0x1.8p+1)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_1030)[name = tensor("new_scale_7")]; + tensor var_1032 = sqrt(x = new_scale_7)[name = tensor("op_1032")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1032)[name = tensor("nkv_1")]; + tensor var_1034_perm_0 = const()[name = tensor("op_1034_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_1355 = transpose(perm = var_1355_perm_0, x = out_19)[name = tensor("transpose_37")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1355)[name = tensor("out_21")]; - tensor var_1359 = const()[name = tensor("op_1359"), val = tensor([1, 3, 256])]; - tensor out_23 = reshape(shape = var_1359, x = out_21)[name = tensor("out_23")]; - tensor var_1361 = silu(x = input_197)[name = tensor("op_1361")]; - tensor input_199 = mul(x = var_1361, y = out_23)[name = tensor("input_199")]; - tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_199)[name = tensor("linear_34")]; - tensor x_33 = add(x = input_193, y = ret_out_7)[name = tensor("x_33")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor var_1034 = transpose(perm = var_1034_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1034)[name = tensor("out_21")]; + tensor var_1038 = const()[name = tensor("op_1038"), val = tensor([1, 3, 256])]; + tensor out_23 = reshape(shape = var_1038, x = out_21)[name = tensor("out_23")]; + tensor var_1040 = silu(x = input_137)[name = tensor("op_1040")]; + tensor input_139 = mul(x = var_1040, y = out_23)[name = tensor("input_139")]; + tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; + tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; + tensor window_25_begin_0 = const()[name = tensor("window_25_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_25_end_0 = const()[name = tensor("window_25_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_25_end_mask_0 = const()[name = tensor("window_25_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_25_squeeze_mask_0 = const()[name = tensor("window_25_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_25 = slice_by_index(begin = window_25_begin_0, end = window_25_end_0, end_mask = window_25_end_mask_0, squeeze_mask = window_25_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_25")]; tensor filled_19_begin_0 = const()[name = tensor("filled_19_begin_0"), val = tensor([3, 0])]; tensor filled_19_end_0 = const()[name = tensor("filled_19_end_0"), val = tensor([4, 1])]; tensor filled_19_end_mask_0 = const()[name = tensor("filled_19_end_mask_0"), val = tensor([false, true])]; tensor filled_19_squeeze_mask_0 = const()[name = tensor("filled_19_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_19 = slice_by_index(begin = filled_19_begin_0, end = filled_19_end_0, end_mask = filled_19_end_mask_0, squeeze_mask = filled_19_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_19")]; - tensor var_1370_begin_0 = const()[name = tensor("op_1370_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1370_end_0 = const()[name = tensor("op_1370_end_0"), val = tensor([1, 1, 256])]; - tensor var_1370_end_mask_0 = const()[name = tensor("op_1370_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1370 = slice_by_index(begin = var_1370_begin_0, end = var_1370_end_0, end_mask = var_1370_end_mask_0, x = x_33)[name = tensor("op_1370")]; - tensor var_1373_begin_0 = const()[name = tensor("op_1373_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1373_end_0 = const()[name = tensor("op_1373_end_0"), val = tensor([1, 16, 256])]; - tensor var_1373_end_mask_0 = const()[name = tensor("op_1373_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1373 = slice_by_index(begin = var_1373_begin_0, end = var_1373_end_0, end_mask = var_1373_end_mask_0, x = window)[name = tensor("op_1373")]; + tensor var_1049_begin_0 = const()[name = tensor("op_1049_begin_0"), val = tensor([0, 0, 0])]; + tensor var_1049_end_0 = const()[name = tensor("op_1049_end_0"), val = tensor([1, 1, 256])]; + tensor var_1049_end_mask_0 = const()[name = tensor("op_1049_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1049 = slice_by_index(begin = var_1049_begin_0, end = var_1049_end_0, end_mask = var_1049_end_mask_0, x = x_21)[name = tensor("op_1049")]; + tensor var_1052_begin_0 = const()[name = tensor("op_1052_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1052_end_0 = const()[name = tensor("op_1052_end_0"), val = tensor([1, 16, 256])]; + tensor var_1052_end_mask_0 = const()[name = tensor("op_1052_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1052 = slice_by_index(begin = var_1052_begin_0, end = var_1052_end_0, end_mask = var_1052_end_mask_0, x = window_25)[name = tensor("op_1052")]; tensor rolled_19_interleave_0 = const()[name = tensor("rolled_19_interleave_0"), val = tensor(false)]; - tensor rolled_19 = concat(axis = var_33, interleave = rolled_19_interleave_0, values = (var_1373, var_1370))[name = tensor("rolled_19")]; + tensor rolled_19 = concat(axis = var_33, interleave = rolled_19_interleave_0, values = (var_1052, var_1049))[name = tensor("rolled_19")]; tensor clip_18 = clip(alpha = var_21, beta = var_20, x = filled_19)[name = tensor("clip_18")]; - tensor var_1378 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_1378")]; - tensor var_1379_dtype_0 = const()[name = tensor("op_1379_dtype_0"), val = tensor("fp32")]; - tensor var_1380 = const()[name = tensor("op_1380"), val = tensor([1, 16, 1])]; - tensor var_1379 = cast(dtype = var_1379_dtype_0, x = var_1378)[name = tensor("cast_141")]; - tensor fill_mask_19 = reshape(shape = var_1380, x = var_1379)[name = tensor("fill_mask_19")]; + tensor var_1057 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_1057")]; + tensor var_1058_dtype_0 = const()[name = tensor("op_1058_dtype_0"), val = tensor("fp32")]; + tensor var_1059 = const()[name = tensor("op_1059"), val = tensor([1, 16, 1])]; + tensor var_1058 = cast(dtype = var_1058_dtype_0, x = var_1057)[name = tensor("cast_141")]; + tensor fill_mask_19 = reshape(shape = var_1059, x = var_1058)[name = tensor("fill_mask_19")]; tensor x_expanded_19_reps_0 = const()[name = tensor("x_expanded_19_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_1370)[name = tensor("x_expanded_19")]; - tensor var_1384 = sub(x = var_17, y = fill_mask_19)[name = tensor("op_1384")]; - tensor var_1385 = mul(x = window, y = var_1384)[name = tensor("op_1385")]; - tensor var_1386 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_1386")]; - tensor filled_window_19 = add(x = var_1385, y = var_1386)[name = tensor("filled_window_19")]; - tensor var_1388 = greater_equal(x = filled_19, y = var_16)[name = tensor("op_1388")]; - tensor var_1389_dtype_0 = const()[name = tensor("op_1389_dtype_0"), val = tensor("fp32")]; - tensor var_1390 = const()[name = tensor("op_1390"), val = tensor([1, 1, 1])]; - tensor var_1389 = cast(dtype = var_1389_dtype_0, x = var_1388)[name = tensor("cast_140")]; - tensor use_roll_19 = reshape(shape = var_1390, x = var_1389)[name = tensor("use_roll_19")]; - tensor var_1392 = sub(x = var_17, y = use_roll_19)[name = tensor("op_1392")]; - tensor var_1393 = mul(x = filled_window_19, y = var_1392)[name = tensor("op_1393")]; - tensor var_1394 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_1394")]; - tensor input_201 = add(x = var_1393, y = var_1394)[name = tensor("input_201")]; - tensor var_1396 = const()[name = tensor("op_1396"), val = tensor(0x1p+0)]; - tensor var_1397 = add(x = filled_19, y = var_1396)[name = tensor("op_1397")]; - tensor clip_19 = clip(alpha = var_21, beta = var_16, x = var_1397)[name = tensor("clip_19")]; - tensor x_35_axes_0 = const()[name = tensor("x_35_axes_0"), val = tensor([-1])]; - tensor x_35 = layer_norm(axes = x_35_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_201)[name = tensor("x_35")]; - tensor input_203_perm_0 = const()[name = tensor("input_203_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_69_pad_type_0 = const()[name = tensor("inputs_69_pad_type_0"), val = tensor("valid")]; - tensor inputs_69_strides_0 = const()[name = tensor("inputs_69_strides_0"), val = tensor([1])]; - tensor inputs_69_pad_0 = const()[name = tensor("inputs_69_pad_0"), val = tensor([0, 0])]; - tensor inputs_69_dilations_0 = const()[name = tensor("inputs_69_dilations_0"), val = tensor([1])]; - tensor inputs_69_groups_0 = const()[name = tensor("inputs_69_groups_0"), val = tensor(1)]; - tensor input_203 = transpose(perm = input_203_perm_0, x = x_35)[name = tensor("transpose_36")]; - tensor inputs_69 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_69_dilations_0, groups = inputs_69_groups_0, pad = inputs_69_pad_0, pad_type = inputs_69_pad_type_0, strides = inputs_69_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_203)[name = tensor("inputs_69")]; - tensor var_1418_split_sizes_0 = const()[name = tensor("op_1418_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1418_axis_0 = const()[name = tensor("op_1418_axis_0"), val = tensor(1)]; - tensor var_1418_0, tensor var_1418_1 = split(axis = var_1418_axis_0, split_sizes = var_1418_split_sizes_0, x = inputs_69)[name = tensor("op_1418")]; - tensor var_1420 = sigmoid(x = var_1418_1)[name = tensor("op_1420")]; - tensor inputs_71 = mul(x = var_1418_0, y = var_1420)[name = tensor("inputs_71")]; - tensor outputs_aug_19_pad_type_0 = const()[name = tensor("outputs_aug_19_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_19_pad_0 = const()[name = tensor("outputs_aug_19_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_19_groups_0 = const()[name = tensor("outputs_aug_19_groups_0"), val = tensor(256)]; - tensor outputs_aug_19_strides_0 = const()[name = tensor("outputs_aug_19_strides_0"), val = tensor([1])]; - tensor outputs_aug_19_dilations_0 = const()[name = tensor("outputs_aug_19_dilations_0"), val = tensor([1])]; - tensor outputs_aug_19 = conv(dilations = outputs_aug_19_dilations_0, groups = outputs_aug_19_groups_0, pad = outputs_aug_19_pad_0, pad_type = outputs_aug_19_pad_type_0, strides = outputs_aug_19_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_71)[name = tensor("outputs_aug_19")]; - tensor input_205_begin_0 = const()[name = tensor("input_205_begin_0"), val = tensor([0, 0, 0])]; - tensor input_205_end_0 = const()[name = tensor("input_205_end_0"), val = tensor([1, 256, 16])]; - tensor input_205_end_mask_0 = const()[name = tensor("input_205_end_mask_0"), val = tensor([true, true, false])]; - tensor input_205 = slice_by_index(begin = input_205_begin_0, end = input_205_end_0, end_mask = input_205_end_mask_0, x = outputs_aug_19)[name = tensor("input_205")]; - tensor inputs_73 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_205)[name = tensor("inputs_73")]; - tensor input_207 = silu(x = inputs_73)[name = tensor("input_207")]; - tensor input_209_pad_type_0 = const()[name = tensor("input_209_pad_type_0"), val = tensor("valid")]; - tensor input_209_strides_0 = const()[name = tensor("input_209_strides_0"), val = tensor([1])]; - tensor input_209_pad_0 = const()[name = tensor("input_209_pad_0"), val = tensor([0, 0])]; - tensor input_209_dilations_0 = const()[name = tensor("input_209_dilations_0"), val = tensor([1])]; - tensor input_209_groups_0 = const()[name = tensor("input_209_groups_0"), val = tensor(1)]; - tensor input_209 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_209_dilations_0, groups = input_209_groups_0, pad = input_209_pad_0, pad_type = input_209_pad_type_0, strides = input_209_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_207)[name = tensor("input_209")]; - tensor conv_out_19_perm_0 = const()[name = tensor("conv_out_19_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1451_begin_0 = const()[name = tensor("op_1451_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1451_end_0 = const()[name = tensor("op_1451_end_0"), val = tensor([1, 16, 256])]; - tensor var_1451_end_mask_0 = const()[name = tensor("op_1451_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_19 = transpose(perm = conv_out_19_perm_0, x = input_209)[name = tensor("transpose_35")]; - tensor var_1451 = slice_by_index(begin = var_1451_begin_0, end = var_1451_end_0, end_mask = var_1451_end_mask_0, x = conv_out_19)[name = tensor("op_1451")]; - tensor var_1454_begin_0 = const()[name = tensor("op_1454_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1454_end_0 = const()[name = tensor("op_1454_end_0"), val = tensor([1, 2, 256])]; - tensor var_1454_end_mask_0 = const()[name = tensor("op_1454_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1454 = slice_by_index(begin = var_1454_begin_0, end = var_1454_end_0, end_mask = var_1454_end_mask_0, x = x_33)[name = tensor("op_1454")]; - tensor var_1457_begin_0 = const()[name = tensor("op_1457_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1457_end_0 = const()[name = tensor("op_1457_end_0"), val = tensor([1, 16, 256])]; - tensor var_1457_end_mask_0 = const()[name = tensor("op_1457_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1457 = slice_by_index(begin = var_1457_begin_0, end = var_1457_end_0, end_mask = var_1457_end_mask_0, x = input_201)[name = tensor("op_1457")]; + tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_1049)[name = tensor("x_expanded_19")]; + tensor var_1063 = sub(x = var_17, y = fill_mask_19)[name = tensor("op_1063")]; + tensor var_1064 = mul(x = window_25, y = var_1063)[name = tensor("op_1064")]; + tensor var_1065 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_1065")]; + tensor filled_window_19 = add(x = var_1064, y = var_1065)[name = tensor("filled_window_19")]; + tensor var_1067 = greater_equal(x = filled_19, y = var_16)[name = tensor("op_1067")]; + tensor var_1068_dtype_0 = const()[name = tensor("op_1068_dtype_0"), val = tensor("fp32")]; + tensor var_1069 = const()[name = tensor("op_1069"), val = tensor([1, 1, 1])]; + tensor var_1068 = cast(dtype = var_1068_dtype_0, x = var_1067)[name = tensor("cast_140")]; + tensor use_roll_19 = reshape(shape = var_1069, x = var_1068)[name = tensor("use_roll_19")]; + tensor var_1071 = sub(x = var_17, y = use_roll_19)[name = tensor("op_1071")]; + tensor var_1072 = mul(x = filled_window_19, y = var_1071)[name = tensor("op_1072")]; + tensor var_1073 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_1073")]; + tensor window_27 = add(x = var_1072, y = var_1073)[name = tensor("window_27")]; + tensor var_1075 = const()[name = tensor("op_1075"), val = tensor(0x1p+0)]; + tensor var_1076 = add(x = filled_19, y = var_1075)[name = tensor("op_1076")]; + tensor clip_19 = clip(alpha = var_21, beta = var_16, x = var_1076)[name = tensor("clip_19")]; + tensor var_1079_begin_0 = const()[name = tensor("op_1079_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1079_end_0 = const()[name = tensor("op_1079_end_0"), val = tensor([1, 2, 256])]; + tensor var_1079_end_mask_0 = const()[name = tensor("op_1079_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1079 = slice_by_index(begin = var_1079_begin_0, end = var_1079_end_0, end_mask = var_1079_end_mask_0, x = x_21)[name = tensor("op_1079")]; + tensor var_1082_begin_0 = const()[name = tensor("op_1082_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1082_end_0 = const()[name = tensor("op_1082_end_0"), val = tensor([1, 16, 256])]; + tensor var_1082_end_mask_0 = const()[name = tensor("op_1082_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1082 = slice_by_index(begin = var_1082_begin_0, end = var_1082_end_0, end_mask = var_1082_end_mask_0, x = window_27)[name = tensor("op_1082")]; tensor rolled_21_interleave_0 = const()[name = tensor("rolled_21_interleave_0"), val = tensor(false)]; - tensor rolled_21 = concat(axis = var_33, interleave = rolled_21_interleave_0, values = (var_1457, var_1454))[name = tensor("rolled_21")]; + tensor rolled_21 = concat(axis = var_33, interleave = rolled_21_interleave_0, values = (var_1082, var_1079))[name = tensor("rolled_21")]; tensor clip_20 = clip(alpha = var_21, beta = var_20, x = clip_19)[name = tensor("clip_20")]; - tensor var_1462 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_1462")]; - tensor var_1463_dtype_0 = const()[name = tensor("op_1463_dtype_0"), val = tensor("fp32")]; - tensor var_1464 = const()[name = tensor("op_1464"), val = tensor([1, 16, 1])]; - tensor var_1463 = cast(dtype = var_1463_dtype_0, x = var_1462)[name = tensor("cast_139")]; - tensor fill_mask_21 = reshape(shape = var_1464, x = var_1463)[name = tensor("fill_mask_21")]; + tensor var_1087 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_1087")]; + tensor var_1088_dtype_0 = const()[name = tensor("op_1088_dtype_0"), val = tensor("fp32")]; + tensor var_1089 = const()[name = tensor("op_1089"), val = tensor([1, 16, 1])]; + tensor var_1088 = cast(dtype = var_1088_dtype_0, x = var_1087)[name = tensor("cast_139")]; + tensor fill_mask_21 = reshape(shape = var_1089, x = var_1088)[name = tensor("fill_mask_21")]; tensor x_expanded_21_reps_0 = const()[name = tensor("x_expanded_21_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_1454)[name = tensor("x_expanded_21")]; - tensor var_1468 = sub(x = var_17, y = fill_mask_21)[name = tensor("op_1468")]; - tensor var_1469 = mul(x = input_201, y = var_1468)[name = tensor("op_1469")]; - tensor var_1470 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_1470")]; - tensor filled_window_21 = add(x = var_1469, y = var_1470)[name = tensor("filled_window_21")]; - tensor var_1472 = greater_equal(x = clip_19, y = var_16)[name = tensor("op_1472")]; - tensor var_1473_dtype_0 = const()[name = tensor("op_1473_dtype_0"), val = tensor("fp32")]; - tensor var_1474 = const()[name = tensor("op_1474"), val = tensor([1, 1, 1])]; - tensor var_1473 = cast(dtype = var_1473_dtype_0, x = var_1472)[name = tensor("cast_138")]; - tensor use_roll_21 = reshape(shape = var_1474, x = var_1473)[name = tensor("use_roll_21")]; - tensor var_1476 = sub(x = var_17, y = use_roll_21)[name = tensor("op_1476")]; - tensor var_1477 = mul(x = filled_window_21, y = var_1476)[name = tensor("op_1477")]; - tensor var_1478 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_1478")]; - tensor input_211 = add(x = var_1477, y = var_1478)[name = tensor("input_211")]; - tensor var_1480 = const()[name = tensor("op_1480"), val = tensor(0x1p+0)]; - tensor var_1481 = add(x = clip_19, y = var_1480)[name = tensor("op_1481")]; - tensor clip_21 = clip(alpha = var_21, beta = var_16, x = var_1481)[name = tensor("clip_21")]; - tensor x_37_axes_0 = const()[name = tensor("x_37_axes_0"), val = tensor([-1])]; - tensor x_37 = layer_norm(axes = x_37_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_211)[name = tensor("x_37")]; - tensor input_213_perm_0 = const()[name = tensor("input_213_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_75_pad_type_0 = const()[name = tensor("inputs_75_pad_type_0"), val = tensor("valid")]; - tensor inputs_75_strides_0 = const()[name = tensor("inputs_75_strides_0"), val = tensor([1])]; - tensor inputs_75_pad_0 = const()[name = tensor("inputs_75_pad_0"), val = tensor([0, 0])]; - tensor inputs_75_dilations_0 = const()[name = tensor("inputs_75_dilations_0"), val = tensor([1])]; - tensor inputs_75_groups_0 = const()[name = tensor("inputs_75_groups_0"), val = tensor(1)]; - tensor input_213 = transpose(perm = input_213_perm_0, x = x_37)[name = tensor("transpose_34")]; - tensor inputs_75 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_75_dilations_0, groups = inputs_75_groups_0, pad = inputs_75_pad_0, pad_type = inputs_75_pad_type_0, strides = inputs_75_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_213)[name = tensor("inputs_75")]; - tensor var_1502_split_sizes_0 = const()[name = tensor("op_1502_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1502_axis_0 = const()[name = tensor("op_1502_axis_0"), val = tensor(1)]; - tensor var_1502_0, tensor var_1502_1 = split(axis = var_1502_axis_0, split_sizes = var_1502_split_sizes_0, x = inputs_75)[name = tensor("op_1502")]; - tensor var_1504 = sigmoid(x = var_1502_1)[name = tensor("op_1504")]; - tensor inputs_77 = mul(x = var_1502_0, y = var_1504)[name = tensor("inputs_77")]; - tensor outputs_aug_21_pad_type_0 = const()[name = tensor("outputs_aug_21_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_21_pad_0 = const()[name = tensor("outputs_aug_21_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_21_groups_0 = const()[name = tensor("outputs_aug_21_groups_0"), val = tensor(256)]; - tensor outputs_aug_21_strides_0 = const()[name = tensor("outputs_aug_21_strides_0"), val = tensor([1])]; - tensor outputs_aug_21_dilations_0 = const()[name = tensor("outputs_aug_21_dilations_0"), val = tensor([1])]; - tensor outputs_aug_21 = conv(dilations = outputs_aug_21_dilations_0, groups = outputs_aug_21_groups_0, pad = outputs_aug_21_pad_0, pad_type = outputs_aug_21_pad_type_0, strides = outputs_aug_21_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_77)[name = tensor("outputs_aug_21")]; - tensor input_215_begin_0 = const()[name = tensor("input_215_begin_0"), val = tensor([0, 0, 0])]; - tensor input_215_end_0 = const()[name = tensor("input_215_end_0"), val = tensor([1, 256, 16])]; - tensor input_215_end_mask_0 = const()[name = tensor("input_215_end_mask_0"), val = tensor([true, true, false])]; - tensor input_215 = slice_by_index(begin = input_215_begin_0, end = input_215_end_0, end_mask = input_215_end_mask_0, x = outputs_aug_21)[name = tensor("input_215")]; - tensor inputs_79 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_215)[name = tensor("inputs_79")]; - tensor input_217 = silu(x = inputs_79)[name = tensor("input_217")]; - tensor input_219_pad_type_0 = const()[name = tensor("input_219_pad_type_0"), val = tensor("valid")]; - tensor input_219_strides_0 = const()[name = tensor("input_219_strides_0"), val = tensor([1])]; - tensor input_219_pad_0 = const()[name = tensor("input_219_pad_0"), val = tensor([0, 0])]; - tensor input_219_dilations_0 = const()[name = tensor("input_219_dilations_0"), val = tensor([1])]; - tensor input_219_groups_0 = const()[name = tensor("input_219_groups_0"), val = tensor(1)]; - tensor input_219 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_219_dilations_0, groups = input_219_groups_0, pad = input_219_pad_0, pad_type = input_219_pad_type_0, strides = input_219_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_217)[name = tensor("input_219")]; - tensor conv_out_21_perm_0 = const()[name = tensor("conv_out_21_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1535_begin_0 = const()[name = tensor("op_1535_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1535_end_0 = const()[name = tensor("op_1535_end_0"), val = tensor([1, 16, 256])]; - tensor var_1535_end_mask_0 = const()[name = tensor("op_1535_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_21 = transpose(perm = conv_out_21_perm_0, x = input_219)[name = tensor("transpose_33")]; - tensor var_1535 = slice_by_index(begin = var_1535_begin_0, end = var_1535_end_0, end_mask = var_1535_end_mask_0, x = conv_out_21)[name = tensor("op_1535")]; - tensor var_1538_begin_0 = const()[name = tensor("op_1538_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1538_end_0 = const()[name = tensor("op_1538_end_0"), val = tensor([1, 1, 256])]; - tensor var_1538_end_mask_0 = const()[name = tensor("op_1538_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1538 = slice_by_index(begin = var_1538_begin_0, end = var_1538_end_0, end_mask = var_1538_end_mask_0, x = x_33)[name = tensor("op_1538")]; - tensor var_1541_begin_0 = const()[name = tensor("op_1541_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1541_end_0 = const()[name = tensor("op_1541_end_0"), val = tensor([1, 16, 256])]; - tensor var_1541_end_mask_0 = const()[name = tensor("op_1541_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1541 = slice_by_index(begin = var_1541_begin_0, end = var_1541_end_0, end_mask = var_1541_end_mask_0, x = input_211)[name = tensor("op_1541")]; + tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_1079)[name = tensor("x_expanded_21")]; + tensor var_1093 = sub(x = var_17, y = fill_mask_21)[name = tensor("op_1093")]; + tensor var_1094 = mul(x = window_27, y = var_1093)[name = tensor("op_1094")]; + tensor var_1095 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_1095")]; + tensor filled_window_21 = add(x = var_1094, y = var_1095)[name = tensor("filled_window_21")]; + tensor var_1097 = greater_equal(x = clip_19, y = var_16)[name = tensor("op_1097")]; + tensor var_1098_dtype_0 = const()[name = tensor("op_1098_dtype_0"), val = tensor("fp32")]; + tensor var_1099 = const()[name = tensor("op_1099"), val = tensor([1, 1, 1])]; + tensor var_1098 = cast(dtype = var_1098_dtype_0, x = var_1097)[name = tensor("cast_138")]; + tensor use_roll_21 = reshape(shape = var_1099, x = var_1098)[name = tensor("use_roll_21")]; + tensor var_1101 = sub(x = var_17, y = use_roll_21)[name = tensor("op_1101")]; + tensor var_1102 = mul(x = filled_window_21, y = var_1101)[name = tensor("op_1102")]; + tensor var_1103 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_1103")]; + tensor window_29 = add(x = var_1102, y = var_1103)[name = tensor("window_29")]; + tensor var_1105 = const()[name = tensor("op_1105"), val = tensor(0x1p+0)]; + tensor var_1106 = add(x = clip_19, y = var_1105)[name = tensor("op_1106")]; + tensor clip_21 = clip(alpha = var_21, beta = var_16, x = var_1106)[name = tensor("clip_21")]; + tensor var_1109_begin_0 = const()[name = tensor("op_1109_begin_0"), val = tensor([0, 2, 0])]; + tensor var_1109_end_0 = const()[name = tensor("op_1109_end_0"), val = tensor([1, 1, 256])]; + tensor var_1109_end_mask_0 = const()[name = tensor("op_1109_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1109 = slice_by_index(begin = var_1109_begin_0, end = var_1109_end_0, end_mask = var_1109_end_mask_0, x = x_21)[name = tensor("op_1109")]; + tensor var_1112_begin_0 = const()[name = tensor("op_1112_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1112_end_0 = const()[name = tensor("op_1112_end_0"), val = tensor([1, 16, 256])]; + tensor var_1112_end_mask_0 = const()[name = tensor("op_1112_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1112 = slice_by_index(begin = var_1112_begin_0, end = var_1112_end_0, end_mask = var_1112_end_mask_0, x = window_29)[name = tensor("op_1112")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1541, var_1538))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1112, var_1109))[name = tensor("rolled")]; tensor clip_22 = clip(alpha = var_21, beta = var_20, x = clip_21)[name = tensor("clip_22")]; - tensor var_1546 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_1546")]; - tensor var_1547_dtype_0 = const()[name = tensor("op_1547_dtype_0"), val = tensor("fp32")]; - tensor var_1548 = const()[name = tensor("op_1548"), val = tensor([1, 16, 1])]; - tensor var_1547 = cast(dtype = var_1547_dtype_0, x = var_1546)[name = tensor("cast_137")]; - tensor fill_mask = reshape(shape = var_1548, x = var_1547)[name = tensor("fill_mask")]; + tensor var_1117 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_1117")]; + tensor var_1118_dtype_0 = const()[name = tensor("op_1118_dtype_0"), val = tensor("fp32")]; + tensor var_1119 = const()[name = tensor("op_1119"), val = tensor([1, 16, 1])]; + tensor var_1118 = cast(dtype = var_1118_dtype_0, x = var_1117)[name = tensor("cast_137")]; + tensor fill_mask = reshape(shape = var_1119, x = var_1118)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1538)[name = tensor("x_expanded")]; - tensor var_1552 = sub(x = var_17, y = fill_mask)[name = tensor("op_1552")]; - tensor var_1553 = mul(x = input_211, y = var_1552)[name = tensor("op_1553")]; - tensor var_1554 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1554")]; - tensor filled_window = add(x = var_1553, y = var_1554)[name = tensor("filled_window")]; - tensor var_1556 = greater_equal(x = clip_21, y = var_16)[name = tensor("op_1556")]; - tensor var_1557_dtype_0 = const()[name = tensor("op_1557_dtype_0"), val = tensor("fp32")]; - tensor var_1558 = const()[name = tensor("op_1558"), val = tensor([1, 1, 1])]; - tensor var_1557 = cast(dtype = var_1557_dtype_0, x = var_1556)[name = tensor("cast_136")]; - tensor use_roll = reshape(shape = var_1558, x = var_1557)[name = tensor("use_roll")]; - tensor var_1560 = sub(x = var_17, y = use_roll)[name = tensor("op_1560")]; - tensor var_1561 = mul(x = filled_window, y = var_1560)[name = tensor("op_1561")]; - tensor var_1562 = mul(x = rolled, y = use_roll)[name = tensor("op_1562")]; - tensor input_221 = add(x = var_1561, y = var_1562)[name = tensor("input_221")]; - tensor var_1564 = const()[name = tensor("op_1564"), val = tensor(0x1p+0)]; - tensor var_1565 = add(x = clip_21, y = var_1564)[name = tensor("op_1565")]; - tensor clip_23 = clip(alpha = var_21, beta = var_16, x = var_1565)[name = tensor("clip_23")]; - tensor x_39_axes_0 = const()[name = tensor("x_39_axes_0"), val = tensor([-1])]; - tensor x_39 = layer_norm(axes = x_39_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_221)[name = tensor("x_39")]; - tensor input_223_perm_0 = const()[name = tensor("input_223_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_81_pad_type_0 = const()[name = tensor("inputs_81_pad_type_0"), val = tensor("valid")]; - tensor inputs_81_strides_0 = const()[name = tensor("inputs_81_strides_0"), val = tensor([1])]; - tensor inputs_81_pad_0 = const()[name = tensor("inputs_81_pad_0"), val = tensor([0, 0])]; - tensor inputs_81_dilations_0 = const()[name = tensor("inputs_81_dilations_0"), val = tensor([1])]; - tensor inputs_81_groups_0 = const()[name = tensor("inputs_81_groups_0"), val = tensor(1)]; - tensor input_223 = transpose(perm = input_223_perm_0, x = x_39)[name = tensor("transpose_32")]; - tensor inputs_81 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_81_dilations_0, groups = inputs_81_groups_0, pad = inputs_81_pad_0, pad_type = inputs_81_pad_type_0, strides = inputs_81_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_223)[name = tensor("inputs_81")]; - tensor var_1586_split_sizes_0 = const()[name = tensor("op_1586_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1586_axis_0 = const()[name = tensor("op_1586_axis_0"), val = tensor(1)]; - tensor var_1586_0, tensor var_1586_1 = split(axis = var_1586_axis_0, split_sizes = var_1586_split_sizes_0, x = inputs_81)[name = tensor("op_1586")]; - tensor var_1588 = sigmoid(x = var_1586_1)[name = tensor("op_1588")]; - tensor inputs_83 = mul(x = var_1586_0, y = var_1588)[name = tensor("inputs_83")]; + tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1109)[name = tensor("x_expanded")]; + tensor var_1123 = sub(x = var_17, y = fill_mask)[name = tensor("op_1123")]; + tensor var_1124 = mul(x = window_29, y = var_1123)[name = tensor("op_1124")]; + tensor var_1125 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1125")]; + tensor filled_window = add(x = var_1124, y = var_1125)[name = tensor("filled_window")]; + tensor var_1127 = greater_equal(x = clip_21, y = var_16)[name = tensor("op_1127")]; + tensor var_1128_dtype_0 = const()[name = tensor("op_1128_dtype_0"), val = tensor("fp32")]; + tensor var_1129 = const()[name = tensor("op_1129"), val = tensor([1, 1, 1])]; + tensor var_1128 = cast(dtype = var_1128_dtype_0, x = var_1127)[name = tensor("cast_136")]; + tensor use_roll = reshape(shape = var_1129, x = var_1128)[name = tensor("use_roll")]; + tensor var_1131 = sub(x = var_17, y = use_roll)[name = tensor("op_1131")]; + tensor var_1132 = mul(x = filled_window, y = var_1131)[name = tensor("op_1132")]; + tensor var_1133 = mul(x = rolled, y = use_roll)[name = tensor("op_1133")]; + tensor window = add(x = var_1132, y = var_1133)[name = tensor("window")]; + tensor var_1135 = const()[name = tensor("op_1135"), val = tensor(0x1p+0)]; + tensor var_1136 = add(x = clip_21, y = var_1135)[name = tensor("op_1136")]; + tensor clip_23 = clip(alpha = var_21, beta = var_16, x = var_1136)[name = tensor("clip_23")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_30, interleave = input_141_interleave_0, values = (window_27, window_29, window))[name = tensor("input_141")]; + tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; + tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; + tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_33_pad_type_0 = const()[name = tensor("inputs_33_pad_type_0"), val = tensor("valid")]; + tensor inputs_33_strides_0 = const()[name = tensor("inputs_33_strides_0"), val = tensor([1])]; + tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; + tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; + tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; + tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; + tensor var_1159_split_sizes_0 = const()[name = tensor("op_1159_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1159_axis_0 = const()[name = tensor("op_1159_axis_0"), val = tensor(1)]; + tensor var_1159_0, tensor var_1159_1 = split(axis = var_1159_axis_0, split_sizes = var_1159_split_sizes_0, x = inputs_33)[name = tensor("op_1159")]; + tensor var_1161 = sigmoid(x = var_1159_1)[name = tensor("op_1161")]; + tensor inputs_35 = mul(x = var_1159_0, y = var_1161)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; tensor outputs_aug_strides_0 = const()[name = tensor("outputs_aug_strides_0"), val = tensor([1])]; tensor outputs_aug_dilations_0 = const()[name = tensor("outputs_aug_dilations_0"), val = tensor([1])]; - tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_83)[name = tensor("outputs_aug")]; - tensor input_225_begin_0 = const()[name = tensor("input_225_begin_0"), val = tensor([0, 0, 0])]; - tensor input_225_end_0 = const()[name = tensor("input_225_end_0"), val = tensor([1, 256, 16])]; - tensor input_225_end_mask_0 = const()[name = tensor("input_225_end_mask_0"), val = tensor([true, true, false])]; - tensor input_225 = slice_by_index(begin = input_225_begin_0, end = input_225_end_0, end_mask = input_225_end_mask_0, x = outputs_aug)[name = tensor("input_225")]; - tensor inputs_85 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_225)[name = tensor("inputs_85")]; - tensor input_227 = silu(x = inputs_85)[name = tensor("input_227")]; - tensor input_229_pad_type_0 = const()[name = tensor("input_229_pad_type_0"), val = tensor("valid")]; - tensor input_229_strides_0 = const()[name = tensor("input_229_strides_0"), val = tensor([1])]; - tensor input_229_pad_0 = const()[name = tensor("input_229_pad_0"), val = tensor([0, 0])]; - tensor input_229_dilations_0 = const()[name = tensor("input_229_dilations_0"), val = tensor([1])]; - tensor input_229_groups_0 = const()[name = tensor("input_229_groups_0"), val = tensor(1)]; - tensor input_229 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_229_dilations_0, groups = input_229_groups_0, pad = input_229_pad_0, pad_type = input_229_pad_type_0, strides = input_229_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_227)[name = tensor("input_229")]; - tensor conv_out_23_perm_0 = const()[name = tensor("conv_out_23_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1619_begin_0 = const()[name = tensor("op_1619_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1619_end_0 = const()[name = tensor("op_1619_end_0"), val = tensor([1, 16, 256])]; - tensor var_1619_end_mask_0 = const()[name = tensor("op_1619_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_23 = transpose(perm = conv_out_23_perm_0, x = input_229)[name = tensor("transpose_31")]; - tensor var_1619 = slice_by_index(begin = var_1619_begin_0, end = var_1619_end_0, end_mask = var_1619_end_mask_0, x = conv_out_23)[name = tensor("op_1619")]; - tensor var_1622_interleave_0 = const()[name = tensor("op_1622_interleave_0"), val = tensor(false)]; - tensor var_1622 = concat(axis = var_33, interleave = var_1622_interleave_0, values = (var_1451, var_1535, var_1619))[name = tensor("op_1622")]; - tensor input_231 = add(x = x_33, y = var_1622)[name = tensor("input_231")]; - tensor input_233_axes_0 = const()[name = tensor("input_233_axes_0"), val = tensor([-1])]; - tensor input_233 = layer_norm(axes = input_233_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_231)[name = tensor("input_233")]; - tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_233)[name = tensor("linear_35")]; - tensor input_235 = silu(x = inputs)[name = tensor("input_235")]; - tensor input_239 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_235)[name = tensor("linear_36")]; - tensor var_1645 = const()[name = tensor("op_1645"), val = tensor(0x1p-1)]; - tensor var_1646 = mul(x = input_239, y = var_1645)[name = tensor("op_1646")]; - tensor input_241 = add(x = var_1646, y = input_231)[name = tensor("input_241")]; - tensor x_41_axes_0 = const()[name = tensor("x_41_axes_0"), val = tensor([-1])]; - tensor x_41 = layer_norm(axes = x_41_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_241)[name = tensor("x_41")]; + tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_35)[name = tensor("outputs_aug")]; + tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; + tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([3, 256, 16])]; + tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; + tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug)[name = tensor("input_145")]; + tensor inputs_37 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_145)[name = tensor("inputs_37")]; + tensor input_147 = silu(x = inputs_37)[name = tensor("input_147")]; + tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; + tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; + tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; + tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; + tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; + tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; + tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1192_begin_0 = const()[name = tensor("op_1192_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1192_end_0 = const()[name = tensor("op_1192_end_0"), val = tensor([3, 16, 256])]; + tensor var_1192_end_mask_0 = const()[name = tensor("op_1192_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_1192 = slice_by_index(begin = var_1192_begin_0, end = var_1192_end_0, end_mask = var_1192_end_mask_0, x = conv_out_7)[name = tensor("op_1192")]; + tensor var_1194_perm_0 = const()[name = tensor("op_1194_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1194 = transpose(perm = var_1194_perm_0, x = var_1192)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_1194)[name = tensor("input_151")]; + tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; + tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; + tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; + tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; + tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; + tensor var_1217 = const()[name = tensor("op_1217"), val = tensor(0x1p-1)]; + tensor var_1218 = mul(x = input_159, y = var_1217)[name = tensor("op_1218")]; + tensor input_161 = add(x = var_1218, y = input_151)[name = tensor("input_161")]; + tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; + tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; tensor cat_interleave_0 = const()[name = tensor("cat_interleave_0"), val = tensor(false)]; - tensor x_ct = transpose(perm = x_ct_perm_0, x = x_41)[name = tensor("transpose_30")]; + tensor x_ct = transpose(perm = x_ct_perm_0, x = x_25)[name = tensor("transpose_30")]; tensor cat = concat(axis = var_27, interleave = cat_interleave_0, values = (cnn_window, x_ct))[name = tensor("cat")]; tensor conv_out_pad_type_0 = const()[name = tensor("conv_out_pad_type_0"), val = tensor("valid")]; tensor conv_out_strides_0 = const()[name = tensor("conv_out_strides_0"), val = tensor([1])]; @@ -1513,44 +1210,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_1664_begin_0 = const()[name = tensor("op_1664_begin_0"), val = tensor([0, 0, 3])]; - tensor var_1664_end_0 = const()[name = tensor("op_1664_end_0"), val = tensor([1, 256, 21])]; - tensor var_1664_end_mask_0 = const()[name = tensor("op_1664_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_1664_begin_0, end = var_1664_end_0, end_mask = var_1664_end_mask_0, x = cat)[name = tensor("op_1664")]; - tensor input_243_perm_0 = const()[name = tensor("input_243_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1666 = const()[name = tensor("op_1666"), val = tensor([-1])]; - tensor input_243 = transpose(perm = input_243_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_1667 = reduce_l2_norm(axes = var_1666, keep_dims = var_36, x = input_243)[name = tensor("op_1667")]; - tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; - tensor clip_24 = clip(alpha = var_12, beta = const_20, x = var_1667)[name = tensor("clip_24")]; - tensor emb = real_div(x = input_243, y = clip_24)[name = tensor("emb")]; - tensor var_1671_axis_0 = const()[name = tensor("op_1671_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_1671_axis_0, values = (var_214, var_594, var_974, nkv_1))[name = tensor("op_1671")]; - tensor var_1673_axis_0 = const()[name = tensor("op_1673_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_1673_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1673")]; - tensor var_1675_axis_0 = const()[name = tensor("op_1675_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_1675_axis_0, values = (input_41, input_101, input_161, input_221))[name = tensor("op_1675")]; - tensor var_1677_axis_0 = const()[name = tensor("op_1677_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_1677_axis_0, values = (clip_5, clip_11, clip_17, clip_23))[name = tensor("op_1677")]; - tensor var_1687 = const()[name = tensor("op_1687"), val = tensor(0x1.5798eep-27)]; - tensor var_1693 = const()[name = tensor("op_1693"), val = tensor(0x1.4f8b58p-17)]; - tensor var_1695 = const()[name = tensor("op_1695"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_1696 = const()[name = tensor("op_1696"), val = tensor(true)]; - tensor var_1698 = const()[name = tensor("op_1698"), val = tensor(0x1p+0)]; - tensor var_1702 = const()[name = tensor("op_1702"), val = tensor(-1)]; - tensor var_1707 = const()[name = tensor("op_1707"), val = tensor(0)]; + tensor var_1236_begin_0 = const()[name = tensor("op_1236_begin_0"), val = tensor([0, 0, 3])]; + tensor var_1236_end_0 = const()[name = tensor("op_1236_end_0"), val = tensor([1, 256, 21])]; + tensor var_1236_end_mask_0 = const()[name = tensor("op_1236_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_1236_begin_0, end = var_1236_end_0, end_mask = var_1236_end_mask_0, x = cat)[name = tensor("op_1236")]; + tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1238 = const()[name = tensor("op_1238"), val = tensor([-1])]; + tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; + tensor var_1239 = reduce_l2_norm(axes = var_1238, keep_dims = var_36, x = input_163)[name = tensor("op_1239")]; + tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; + tensor clip_24 = clip(alpha = var_12, beta = const_12, x = var_1239)[name = tensor("clip_24")]; + tensor emb = real_div(x = input_163, y = clip_24)[name = tensor("emb")]; + tensor var_1243_axis_0 = const()[name = tensor("op_1243_axis_0"), val = tensor(0)]; + tensor enc_kv_new = stack(axis = var_1243_axis_0, values = (var_214, var_487, var_760, nkv_1))[name = tensor("op_1243")]; + tensor var_1245_axis_0 = const()[name = tensor("op_1245_axis_0"), val = tensor(0)]; + tensor enc_scale_new = stack(axis = var_1245_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1245")]; + tensor var_1247_axis_0 = const()[name = tensor("op_1247_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1247_axis_0, values = (window_7, window_15, window_23, window))[name = tensor("op_1247")]; + tensor var_1249_axis_0 = const()[name = tensor("op_1249_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1249_axis_0, values = (clip_5, clip_11, clip_17, clip_23))[name = tensor("op_1249")]; + tensor var_1259 = const()[name = tensor("op_1259"), val = tensor(0x1.5798eep-27)]; + tensor var_1264 = const()[name = tensor("op_1264"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1266 = const()[name = tensor("op_1266"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1267 = const()[name = tensor("op_1267"), val = tensor(true)]; + tensor var_1269 = const()[name = tensor("op_1269"), val = tensor(0x1p+0)]; + tensor var_1273 = const()[name = tensor("op_1273"), val = tensor(-1)]; + tensor var_1279 = const()[name = tensor("op_1279"), val = tensor(0)]; tensor pos = const()[name = tensor("pos"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395456)))]; - tensor var_1769_axes_0 = const()[name = tensor("op_1769_axes_0"), val = tensor([2])]; - tensor var_1769 = expand_dims(axes = var_1769_axes_0, x = emb)[name = tensor("op_1769")]; - tensor var_1770 = const()[name = tensor("op_1770"), val = tensor([1, 1, 12, 1])]; - tensor emb_exp = tile(reps = var_1770, x = var_1769)[name = tensor("emb_exp")]; - tensor input_245_interleave_0 = const()[name = tensor("input_245_interleave_0"), val = tensor(false)]; - tensor input_245 = concat(axis = var_1702, interleave = input_245_interleave_0, values = (emb_exp, pos))[name = tensor("input_245")]; - tensor x_43 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_245)[name = tensor("linear_37")]; - tensor var_1777_perm_0 = const()[name = tensor("op_1777_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1781 = const()[name = tensor("op_1781"), val = tensor([12, 3, 256])]; - tensor var_1777 = transpose(perm = var_1777_perm_0, x = x_43)[name = tensor("transpose_28")]; - tensor x_45 = reshape(shape = var_1781, x = var_1777)[name = tensor("x_45")]; + tensor var_1341_axes_0 = const()[name = tensor("op_1341_axes_0"), val = tensor([2])]; + tensor var_1341 = expand_dims(axes = var_1341_axes_0, x = emb)[name = tensor("op_1341")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 12, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1341)[name = tensor("emb_exp")]; + tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; + tensor input_165 = concat(axis = var_1273, interleave = input_165_interleave_0, values = (emb_exp, pos))[name = tensor("input_165")]; + tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; + tensor var_1349_perm_0 = const()[name = tensor("op_1349_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1353 = const()[name = tensor("op_1353"), val = tensor([12, 3, 256])]; + tensor var_1349 = transpose(perm = var_1349_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1353, x = var_1349)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 12, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1561,132 +1258,132 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_1789 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_45)[name = tensor("linear_38")]; - tensor var_1790 = const()[name = tensor("op_1790"), val = tensor([12, 3, 4, 64])]; - tensor var_1791 = reshape(shape = var_1790, x = var_1789)[name = tensor("op_1791")]; + tensor var_1361 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1362 = const()[name = tensor("op_1362"), val = tensor([12, 3, 4, 64])]; + tensor var_1363 = reshape(shape = var_1362, x = var_1361)[name = tensor("op_1363")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1795 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_45)[name = tensor("linear_39")]; - tensor var_1796 = const()[name = tensor("op_1796"), val = tensor(0x1p-3)]; - tensor var_1797 = mul(x = var_1795, y = var_1796)[name = tensor("op_1797")]; - tensor var_1798 = const()[name = tensor("op_1798"), val = tensor([12, 3, 4, 64])]; - tensor var_1799 = reshape(shape = var_1798, x = var_1797)[name = tensor("op_1799")]; + tensor var_1367 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1368 = const()[name = tensor("op_1368"), val = tensor(0x1p-3)]; + tensor var_1369 = mul(x = var_1367, y = var_1368)[name = tensor("op_1369")]; + tensor var_1370 = const()[name = tensor("op_1370"), val = tensor([12, 3, 4, 64])]; + tensor var_1371 = reshape(shape = var_1370, x = var_1369)[name = tensor("op_1371")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1803 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_45)[name = tensor("linear_40")]; - tensor var_1804 = const()[name = tensor("op_1804"), val = tensor([12, 3, 4, 64])]; - tensor var_1805 = reshape(shape = var_1804, x = var_1803)[name = tensor("op_1805")]; + tensor var_1375 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1376 = const()[name = tensor("op_1376"), val = tensor([12, 3, 4, 64])]; + tensor var_1377 = reshape(shape = var_1376, x = var_1375)[name = tensor("op_1377")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_249 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_45)[name = tensor("linear_41")]; + tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_1707, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1279, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; - tensor const_28 = const()[name = tensor("const_28"), val = tensor(0x1.fffffep+127)]; - tensor clip_25 = clip(alpha = var_1698, beta = const_28, x = s_t_9)[name = tensor("clip_25")]; + tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; + tensor clip_25 = clip(alpha = var_1269, beta = const_20, x = s_t_9)[name = tensor("clip_25")]; tensor sqrt_s_t_9 = sqrt(x = clip_25)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_1799)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_1791)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1371)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1363)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_1817 = const()[name = tensor("op_1817"), val = tensor([1, 3])]; - tensor var_1818 = reshape(shape = var_1817, x = valid_mask)[name = tensor("op_1818")]; - tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1818)[name = tensor("causal_with_valid_1")]; - tensor var_1820 = const()[name = tensor("op_1820"), val = tensor([3, 1])]; - tensor var_1821 = reshape(shape = var_1820, x = sqrt_s_t_9)[name = tensor("op_1821")]; - tensor M_9 = real_div(x = causal_with_valid_1, y = var_1821)[name = tensor("M_9")]; - tensor var_1823 = mul(x = qk_9, y = M_9)[name = tensor("op_1823")]; + tensor var_1389 = const()[name = tensor("op_1389"), val = tensor([1, 3])]; + tensor var_1390 = reshape(shape = var_1389, x = valid_mask)[name = tensor("op_1390")]; + tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1390)[name = tensor("causal_with_valid_1")]; + tensor var_1392 = const()[name = tensor("op_1392"), val = tensor([3, 1])]; + tensor var_1393 = reshape(shape = var_1392, x = sqrt_s_t_9)[name = tensor("op_1393")]; + tensor M_9 = real_div(x = causal_with_valid_1, y = var_1393)[name = tensor("M_9")]; + tensor var_1395 = mul(x = qk_9, y = M_9)[name = tensor("op_1395")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_1805)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1823, y = v_9)[name = tensor("inner_9")]; - tensor var_1825_transpose_x_0 = const()[name = tensor("op_1825_transpose_x_0"), val = tensor(false)]; - tensor var_1825_transpose_y_0 = const()[name = tensor("op_1825_transpose_y_0"), val = tensor(false)]; - tensor var_1825 = matmul(transpose_x = var_1825_transpose_x_0, transpose_y = var_1825_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1825")]; - tensor var_1826 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1826")]; - tensor var_1827 = const()[name = tensor("op_1827"), val = tensor([1, 1, 3, 1])]; - tensor var_1828 = reshape(shape = var_1827, x = var_1826)[name = tensor("op_1828")]; - tensor cross_9 = mul(x = var_1825, y = var_1828)[name = tensor("cross_9")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1377)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1395, y = v_9)[name = tensor("inner_9")]; + tensor var_1397_transpose_x_0 = const()[name = tensor("op_1397_transpose_x_0"), val = tensor(false)]; + tensor var_1397_transpose_y_0 = const()[name = tensor("op_1397_transpose_y_0"), val = tensor(false)]; + tensor var_1397 = matmul(transpose_x = var_1397_transpose_x_0, transpose_y = var_1397_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1397")]; + tensor var_1398 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1398")]; + tensor var_1399 = const()[name = tensor("op_1399"), val = tensor([1, 1, 3, 1])]; + tensor var_1400 = reshape(shape = var_1399, x = var_1398)[name = tensor("op_1400")]; + tensor cross_9 = mul(x = var_1397, y = var_1400)[name = tensor("cross_9")]; tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; - tensor var_1831 = const()[name = tensor("op_1831"), val = tensor([1, 1, 3, 1])]; - tensor var_1832 = reshape(shape = var_1831, x = valid_mask)[name = tensor("op_1832")]; - tensor v_masked_1 = mul(x = v_9, y = var_1832)[name = tensor("v_masked_1")]; - tensor var_1834 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1834")]; - tensor var_1836_transpose_x_1 = const()[name = tensor("op_1836_transpose_x_1"), val = tensor(true)]; - tensor var_1836_transpose_y_1 = const()[name = tensor("op_1836_transpose_y_1"), val = tensor(false)]; - tensor var_1836 = matmul(transpose_x = var_1836_transpose_x_1, transpose_y = var_1836_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1836")]; - tensor new_kv_unnorm_9 = add(x = var_1834, y = var_1836)[name = tensor("new_kv_unnorm_9")]; - tensor var_1838_keep_dims_0 = const()[name = tensor("op_1838_keep_dims_0"), val = tensor(false)]; - tensor var_1838 = reduce_sum(keep_dims = var_1838_keep_dims_0, x = valid_mask)[name = tensor("op_1838")]; - tensor var_1839 = const()[name = tensor("op_1839"), val = tensor([1])]; - tensor var_1840 = reshape(shape = var_1839, x = var_1838)[name = tensor("op_1840")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_1840)[name = tensor("new_scale_9")]; - tensor const_29 = const()[name = tensor("const_29"), val = tensor(0x1.fffffep+127)]; - tensor clip_26 = clip(alpha = var_1698, beta = const_29, x = new_scale_9)[name = tensor("clip_26")]; + tensor var_1403 = const()[name = tensor("op_1403"), val = tensor([1, 1, 3, 1])]; + tensor var_1404 = reshape(shape = var_1403, x = valid_mask)[name = tensor("op_1404")]; + tensor v_masked_1 = mul(x = v_9, y = var_1404)[name = tensor("v_masked_1")]; + tensor var_1406 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1406")]; + tensor var_1408_transpose_x_1 = const()[name = tensor("op_1408_transpose_x_1"), val = tensor(true)]; + tensor var_1408_transpose_y_1 = const()[name = tensor("op_1408_transpose_y_1"), val = tensor(false)]; + tensor var_1408 = matmul(transpose_x = var_1408_transpose_x_1, transpose_y = var_1408_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1408")]; + tensor new_kv_unnorm_9 = add(x = var_1406, y = var_1408)[name = tensor("new_kv_unnorm_9")]; + tensor var_1410_keep_dims_0 = const()[name = tensor("op_1410_keep_dims_0"), val = tensor(false)]; + tensor var_1410 = reduce_sum(keep_dims = var_1410_keep_dims_0, x = valid_mask)[name = tensor("op_1410")]; + tensor var_1411 = const()[name = tensor("op_1411"), val = tensor([1])]; + tensor var_1412 = reshape(shape = var_1411, x = var_1410)[name = tensor("op_1412")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1412)[name = tensor("new_scale_9")]; + tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; + tensor clip_26 = clip(alpha = var_1269, beta = const_21, x = new_scale_9)[name = tensor("clip_26")]; tensor sqrt_new_scale_1 = sqrt(x = clip_26)[name = tensor("sqrt_new_scale_1")]; - tensor var_1844 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1844")]; - tensor var_1845_perm_0 = const()[name = tensor("op_1845_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1416 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1416")]; + tensor var_1417_perm_0 = const()[name = tensor("op_1417_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_1845 = transpose(perm = var_1845_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1695, x = var_1845)[name = tensor("out_27")]; - tensor var_1849 = const()[name = tensor("op_1849"), val = tensor([12, 3, 256])]; - tensor out_29 = reshape(shape = var_1849, x = out_27)[name = tensor("out_29")]; - tensor var_1851 = silu(x = input_249)[name = tensor("op_1851")]; - tensor input_251 = mul(x = var_1851, y = out_29)[name = tensor("input_251")]; - tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_251)[name = tensor("linear_42")]; - tensor input_253 = add(x = x_45, y = ret_out_9)[name = tensor("input_253")]; + tensor var_1417 = transpose(perm = var_1417_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1266, x = var_1417)[name = tensor("out_27")]; + tensor var_1421 = const()[name = tensor("op_1421"), val = tensor([12, 3, 256])]; + tensor out_29 = reshape(shape = var_1421, x = out_27)[name = tensor("out_29")]; + tensor var_1423 = silu(x = input_169)[name = tensor("op_1423")]; + tensor input_171 = mul(x = var_1423, y = out_29)[name = tensor("input_171")]; + tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; + tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1693, gamma = decoder_norm11_0_weight, x = input_253)[name = tensor("xt_1")]; - tensor var_1861 = const()[name = tensor("op_1861"), val = tensor([1, 12, 3, 256])]; - tensor var_1862 = reshape(shape = var_1861, x = xt_1)[name = tensor("op_1862")]; - tensor var_1863_perm_0 = const()[name = tensor("op_1863_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1866 = const()[name = tensor("op_1866"), val = tensor([3, 12, 256])]; - tensor var_1863 = transpose(perm = var_1863_perm_0, x = var_1862)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_1866, x = var_1863)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1264, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1433 = const()[name = tensor("op_1433"), val = tensor([1, 12, 3, 256])]; + tensor var_1434 = reshape(shape = var_1433, x = xt_1)[name = tensor("op_1434")]; + tensor var_1435_perm_0 = const()[name = tensor("op_1435_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1438 = const()[name = tensor("op_1438"), val = tensor([3, 12, 256])]; + tensor var_1435 = transpose(perm = var_1435_perm_0, x = var_1434)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1438, x = var_1435)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_1889 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_12 = const()[name = tensor("concat_12"), val = tensor([12, 3, 3, 256])]; - tensor var_1891 = reshape(shape = concat_12, x = var_1889)[name = tensor("op_1891")]; - tensor var_1892_axes_0 = const()[name = tensor("op_1892_axes_0"), val = tensor([0])]; - tensor var_1892 = expand_dims(axes = var_1892_axes_0, x = var_1891)[name = tensor("op_1892")]; - tensor var_1893_perm_0 = const()[name = tensor("op_1893_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1894_axes_0 = const()[name = tensor("op_1894_axes_0"), val = tensor([-2])]; - tensor var_1893 = transpose(perm = var_1893_perm_0, x = var_1892)[name = tensor("transpose_21")]; - tensor var_1894 = squeeze(axes = var_1894_axes_0, x = var_1893)[name = tensor("op_1894")]; + tensor var_1461 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_13 = const()[name = tensor("concat_13"), val = tensor([12, 3, 3, 256])]; + tensor var_1463 = reshape(shape = concat_13, x = var_1461)[name = tensor("op_1463")]; + tensor var_1464_axes_0 = const()[name = tensor("op_1464_axes_0"), val = tensor([0])]; + tensor var_1464 = expand_dims(axes = var_1464_axes_0, x = var_1463)[name = tensor("op_1464")]; + tensor var_1465_perm_0 = const()[name = tensor("op_1465_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1466_axes_0 = const()[name = tensor("op_1466_axes_0"), val = tensor([-2])]; + tensor var_1465 = transpose(perm = var_1465_perm_0, x = var_1464)[name = tensor("transpose_21")]; + tensor var_1466 = squeeze(axes = var_1466_axes_0, x = var_1465)[name = tensor("op_1466")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 12, 3, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1894)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1466)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 12, 3, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1894)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1466)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 12, 3, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1894)[name = tensor("v_11")]; - tensor var_1902 = const()[name = tensor("op_1902"), val = tensor([12, 12, 64])]; - tensor var_1903 = reshape(shape = var_1902, x = q_11)[name = tensor("op_1903")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1466)[name = tensor("v_11")]; + tensor var_1474 = const()[name = tensor("op_1474"), val = tensor([12, 12, 64])]; + tensor var_1475 = reshape(shape = var_1474, x = q_11)[name = tensor("op_1475")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1909 = const()[name = tensor("op_1909"), val = tensor([12, 12, 64])]; - tensor var_1910 = reshape(shape = var_1909, x = k_11)[name = tensor("op_1910")]; + tensor var_1481 = const()[name = tensor("op_1481"), val = tensor([12, 12, 64])]; + tensor var_1482 = reshape(shape = var_1481, x = k_11)[name = tensor("op_1482")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1916 = const()[name = tensor("op_1916"), val = tensor([12, 12, 64])]; - tensor var_1917 = reshape(shape = var_1916, x = v_11)[name = tensor("op_1917")]; + tensor var_1488 = const()[name = tensor("op_1488"), val = tensor([12, 12, 64])]; + tensor var_1489 = reshape(shape = var_1488, x = v_11)[name = tensor("op_1489")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1920 = const()[name = tensor("op_1920"), val = tensor([3, 4, 12, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_1903)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_1920, x = q_13)[name = tensor("q_15")]; - tensor var_1922 = const()[name = tensor("op_1922"), val = tensor([3, 4, 12, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_1910)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_1922, x = k_13)[name = tensor("k_15")]; - tensor var_1924 = const()[name = tensor("op_1924"), val = tensor([3, 4, 12, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_1917)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_1924, x = v_13)[name = tensor("v_15")]; + tensor var_1492 = const()[name = tensor("op_1492"), val = tensor([3, 4, 12, 64])]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1475)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1492, x = q_13)[name = tensor("q_15")]; + tensor var_1494 = const()[name = tensor("op_1494"), val = tensor([3, 4, 12, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1482)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1494, x = k_13)[name = tensor("k_15")]; + tensor var_1496 = const()[name = tensor("op_1496"), val = tensor([3, 4, 12, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1489)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1496, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -1697,30 +1394,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_1927 = const()[name = tensor("op_1927"), val = tensor([2, 0, 1, 3])]; - tensor var_1932 = const()[name = tensor("op_1932"), val = tensor([36, 256])]; - tensor var_1928 = transpose(perm = var_1927, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_1932, x = var_1928)[name = tensor("attn_output_3")]; + tensor var_1499 = const()[name = tensor("op_1499"), val = tensor([2, 0, 1, 3])]; + tensor var_1504 = const()[name = tensor("op_1504"), val = tensor([36, 256])]; + tensor var_1500 = transpose(perm = var_1499, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1504, x = var_1500)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_1936 = const()[name = tensor("op_1936"), val = tensor([12, 3, 256])]; - tensor attn_output_7 = reshape(shape = var_1936, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1508 = const()[name = tensor("op_1508"), val = tensor([12, 3, 256])]; + tensor attn_output_7 = reshape(shape = var_1508, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; - tensor input_255 = add(x = query_1, y = sa2_out_1)[name = tensor("input_255")]; - tensor input_257_axes_0 = const()[name = tensor("input_257_axes_0"), val = tensor([-1])]; - tensor input_257 = layer_norm(axes = input_257_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1693, gamma = decoder_norm21_0_weight, x = input_255)[name = tensor("input_257")]; - tensor input_259 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_257)[name = tensor("linear_45")]; - tensor input_261 = relu(x = input_259)[name = tensor("input_261")]; - tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_261)[name = tensor("linear_46")]; - tensor input_263 = add(x = input_257, y = ffn_out_1)[name = tensor("input_263")]; + tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; + tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1264, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; + tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; + tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; + tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1693, gamma = decoder_norm22_0_weight, x = input_263)[name = tensor("xt_3")]; - tensor var_1956 = const()[name = tensor("op_1956"), val = tensor([1, 3, 12, 256])]; - tensor x_47 = reshape(shape = var_1956, x = xt_3)[name = tensor("x_47")]; - tensor var_1958_perm_0 = const()[name = tensor("op_1958_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1962 = const()[name = tensor("op_1962"), val = tensor([12, 3, 256])]; - tensor var_1958 = transpose(perm = var_1958_perm_0, x = x_47)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_1962, x = var_1958)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1264, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1528 = const()[name = tensor("op_1528"), val = tensor([1, 3, 12, 256])]; + tensor x_31 = reshape(shape = var_1528, x = xt_3)[name = tensor("x_31")]; + tensor var_1530_perm_0 = const()[name = tensor("op_1530_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1534 = const()[name = tensor("op_1534"), val = tensor([12, 3, 256])]; + tensor var_1530 = transpose(perm = var_1530_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1534, x = var_1530)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 12, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1731,120 +1428,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_1970 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_1971 = const()[name = tensor("op_1971"), val = tensor([12, 3, 4, 64])]; - tensor var_1972 = reshape(shape = var_1971, x = var_1970)[name = tensor("op_1972")]; + tensor var_1542 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1543 = const()[name = tensor("op_1543"), val = tensor([12, 3, 4, 64])]; + tensor var_1544 = reshape(shape = var_1543, x = var_1542)[name = tensor("op_1544")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1976 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_1977 = const()[name = tensor("op_1977"), val = tensor(0x1p-3)]; - tensor var_1978 = mul(x = var_1976, y = var_1977)[name = tensor("op_1978")]; - tensor var_1979 = const()[name = tensor("op_1979"), val = tensor([12, 3, 4, 64])]; - tensor var_1980 = reshape(shape = var_1979, x = var_1978)[name = tensor("op_1980")]; + tensor var_1548 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1549 = const()[name = tensor("op_1549"), val = tensor(0x1p-3)]; + tensor var_1550 = mul(x = var_1548, y = var_1549)[name = tensor("op_1550")]; + tensor var_1551 = const()[name = tensor("op_1551"), val = tensor([12, 3, 4, 64])]; + tensor var_1552 = reshape(shape = var_1551, x = var_1550)[name = tensor("op_1552")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1984 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_1985 = const()[name = tensor("op_1985"), val = tensor([12, 3, 4, 64])]; - tensor var_1986 = reshape(shape = var_1985, x = var_1984)[name = tensor("op_1986")]; + tensor var_1556 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1557 = const()[name = tensor("op_1557"), val = tensor([12, 3, 4, 64])]; + tensor var_1558 = reshape(shape = var_1557, x = var_1556)[name = tensor("op_1558")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_267 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; + tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; - tensor const_40 = const()[name = tensor("const_40"), val = tensor(0x1.fffffep+127)]; - tensor clip_27 = clip(alpha = var_1698, beta = const_40, x = s_t)[name = tensor("clip_27")]; + tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; + tensor clip_27 = clip(alpha = var_1269, beta = const_32, x = s_t)[name = tensor("clip_27")]; tensor sqrt_s_t = sqrt(x = clip_27)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_1980)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_1972)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1552)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1544)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_2001 = const()[name = tensor("op_2001"), val = tensor([3, 1])]; - tensor var_2002 = reshape(shape = var_2001, x = sqrt_s_t)[name = tensor("op_2002")]; - tensor M = real_div(x = causal_with_valid_1, y = var_2002)[name = tensor("M")]; - tensor var_2004 = mul(x = qk, y = M)[name = tensor("op_2004")]; + tensor var_1573 = const()[name = tensor("op_1573"), val = tensor([3, 1])]; + tensor var_1574 = reshape(shape = var_1573, x = sqrt_s_t)[name = tensor("op_1574")]; + tensor M = real_div(x = causal_with_valid_1, y = var_1574)[name = tensor("M")]; + tensor var_1576 = mul(x = qk, y = M)[name = tensor("op_1576")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_1986)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_2004, y = v_17)[name = tensor("inner")]; - tensor var_2006_transpose_x_0 = const()[name = tensor("op_2006_transpose_x_0"), val = tensor(false)]; - tensor var_2006_transpose_y_0 = const()[name = tensor("op_2006_transpose_y_0"), val = tensor(false)]; - tensor var_2006 = matmul(transpose_x = var_2006_transpose_x_0, transpose_y = var_2006_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_2006")]; - tensor var_2007 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_2007")]; - tensor var_2008 = const()[name = tensor("op_2008"), val = tensor([1, 1, 3, 1])]; - tensor var_2009 = reshape(shape = var_2008, x = var_2007)[name = tensor("op_2009")]; - tensor cross = mul(x = var_2006, y = var_2009)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1558)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1576, y = v_17)[name = tensor("inner")]; + tensor var_1578_transpose_x_0 = const()[name = tensor("op_1578_transpose_x_0"), val = tensor(false)]; + tensor var_1578_transpose_y_0 = const()[name = tensor("op_1578_transpose_y_0"), val = tensor(false)]; + tensor var_1578 = matmul(transpose_x = var_1578_transpose_x_0, transpose_y = var_1578_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1578")]; + tensor var_1579 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1579")]; + tensor var_1580 = const()[name = tensor("op_1580"), val = tensor([1, 1, 3, 1])]; + tensor var_1581 = reshape(shape = var_1580, x = var_1579)[name = tensor("op_1581")]; + tensor cross = mul(x = var_1578, y = var_1581)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_1832)[name = tensor("v_masked")]; - tensor var_2015 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_2015")]; - tensor var_2017_transpose_x_1 = const()[name = tensor("op_2017_transpose_x_1"), val = tensor(true)]; - tensor var_2017_transpose_y_1 = const()[name = tensor("op_2017_transpose_y_1"), val = tensor(false)]; - tensor var_2017 = matmul(transpose_x = var_2017_transpose_x_1, transpose_y = var_2017_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_2017")]; - tensor new_kv_unnorm = add(x = var_2015, y = var_2017)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_1840)[name = tensor("new_scale")]; - tensor const_41 = const()[name = tensor("const_41"), val = tensor(0x1.fffffep+127)]; - tensor clip_28 = clip(alpha = var_1698, beta = const_41, x = new_scale)[name = tensor("clip_28")]; + tensor v_masked = mul(x = v_17, y = var_1404)[name = tensor("v_masked")]; + tensor var_1587 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1587")]; + tensor var_1589_transpose_x_1 = const()[name = tensor("op_1589_transpose_x_1"), val = tensor(true)]; + tensor var_1589_transpose_y_1 = const()[name = tensor("op_1589_transpose_y_1"), val = tensor(false)]; + tensor var_1589 = matmul(transpose_x = var_1589_transpose_x_1, transpose_y = var_1589_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1589")]; + tensor new_kv_unnorm = add(x = var_1587, y = var_1589)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1412)[name = tensor("new_scale")]; + tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; + tensor clip_28 = clip(alpha = var_1269, beta = const_33, x = new_scale)[name = tensor("clip_28")]; tensor sqrt_new_scale = sqrt(x = clip_28)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_2026_perm_0 = const()[name = tensor("op_2026_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1598_perm_0 = const()[name = tensor("op_1598_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_2026 = transpose(perm = var_2026_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1695, x = var_2026)[name = tensor("out_33")]; - tensor var_2030 = const()[name = tensor("op_2030"), val = tensor([12, 3, 256])]; - tensor out = reshape(shape = var_2030, x = out_33)[name = tensor("out")]; - tensor var_2032 = silu(x = input_267)[name = tensor("op_2032")]; - tensor input_269 = mul(x = var_2032, y = out)[name = tensor("input_269")]; - tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_269)[name = tensor("linear_51")]; - tensor input_271 = add(x = x, y = ret_out)[name = tensor("input_271")]; + tensor var_1598 = transpose(perm = var_1598_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1266, x = var_1598)[name = tensor("out_33")]; + tensor var_1602 = const()[name = tensor("op_1602"), val = tensor([12, 3, 256])]; + tensor out = reshape(shape = var_1602, x = out_33)[name = tensor("out")]; + tensor var_1604 = silu(x = input_187)[name = tensor("op_1604")]; + tensor input_189 = mul(x = var_1604, y = out)[name = tensor("input_189")]; + tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; + tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1693, gamma = decoder_norm11_1_weight, x = input_271)[name = tensor("xt_5")]; - tensor var_2042 = const()[name = tensor("op_2042"), val = tensor([1, 12, 3, 256])]; - tensor var_2043 = reshape(shape = var_2042, x = xt_5)[name = tensor("op_2043")]; - tensor var_2044_perm_0 = const()[name = tensor("op_2044_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2047 = const()[name = tensor("op_2047"), val = tensor([3, 12, 256])]; - tensor var_2044 = transpose(perm = var_2044_perm_0, x = var_2043)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_2047, x = var_2044)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1264, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1614 = const()[name = tensor("op_1614"), val = tensor([1, 12, 3, 256])]; + tensor var_1615 = reshape(shape = var_1614, x = xt_5)[name = tensor("op_1615")]; + tensor var_1616_perm_0 = const()[name = tensor("op_1616_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1619 = const()[name = tensor("op_1619"), val = tensor([3, 12, 256])]; + tensor var_1616 = transpose(perm = var_1616_perm_0, x = var_1615)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1619, x = var_1616)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_2070 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_13 = const()[name = tensor("concat_13"), val = tensor([12, 3, 3, 256])]; - tensor var_2072 = reshape(shape = concat_13, x = var_2070)[name = tensor("op_2072")]; - tensor var_2073_axes_0 = const()[name = tensor("op_2073_axes_0"), val = tensor([0])]; - tensor var_2073 = expand_dims(axes = var_2073_axes_0, x = var_2072)[name = tensor("op_2073")]; - tensor var_2074_perm_0 = const()[name = tensor("op_2074_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2075_axes_0 = const()[name = tensor("op_2075_axes_0"), val = tensor([-2])]; - tensor var_2074 = transpose(perm = var_2074_perm_0, x = var_2073)[name = tensor("transpose_8")]; - tensor var_2075 = squeeze(axes = var_2075_axes_0, x = var_2074)[name = tensor("op_2075")]; + tensor var_1642 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_14 = const()[name = tensor("concat_14"), val = tensor([12, 3, 3, 256])]; + tensor var_1644 = reshape(shape = concat_14, x = var_1642)[name = tensor("op_1644")]; + tensor var_1645_axes_0 = const()[name = tensor("op_1645_axes_0"), val = tensor([0])]; + tensor var_1645 = expand_dims(axes = var_1645_axes_0, x = var_1644)[name = tensor("op_1645")]; + tensor var_1646_perm_0 = const()[name = tensor("op_1646_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1647_axes_0 = const()[name = tensor("op_1647_axes_0"), val = tensor([-2])]; + tensor var_1646 = transpose(perm = var_1646_perm_0, x = var_1645)[name = tensor("transpose_8")]; + tensor var_1647 = squeeze(axes = var_1647_axes_0, x = var_1646)[name = tensor("op_1647")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 12, 3, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_2075)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1647)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 12, 3, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_2075)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1647)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 12, 3, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_2075)[name = tensor("v_19")]; - tensor var_2083 = const()[name = tensor("op_2083"), val = tensor([12, 12, 64])]; - tensor var_2084 = reshape(shape = var_2083, x = q_19)[name = tensor("op_2084")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1647)[name = tensor("v_19")]; + tensor var_1655 = const()[name = tensor("op_1655"), val = tensor([12, 12, 64])]; + tensor var_1656 = reshape(shape = var_1655, x = q_19)[name = tensor("op_1656")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2090 = const()[name = tensor("op_2090"), val = tensor([12, 12, 64])]; - tensor var_2091 = reshape(shape = var_2090, x = k_19)[name = tensor("op_2091")]; + tensor var_1662 = const()[name = tensor("op_1662"), val = tensor([12, 12, 64])]; + tensor var_1663 = reshape(shape = var_1662, x = k_19)[name = tensor("op_1663")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2097 = const()[name = tensor("op_2097"), val = tensor([12, 12, 64])]; - tensor var_2098 = reshape(shape = var_2097, x = v_19)[name = tensor("op_2098")]; + tensor var_1669 = const()[name = tensor("op_1669"), val = tensor([12, 12, 64])]; + tensor var_1670 = reshape(shape = var_1669, x = v_19)[name = tensor("op_1670")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2101 = const()[name = tensor("op_2101"), val = tensor([3, 4, 12, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_2084)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_2101, x = q_21)[name = tensor("q")]; - tensor var_2103 = const()[name = tensor("op_2103"), val = tensor([3, 4, 12, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_2091)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_2103, x = k_21)[name = tensor("k")]; - tensor var_2105 = const()[name = tensor("op_2105"), val = tensor([3, 4, 12, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_2098)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_2105, x = v_21)[name = tensor("v")]; + tensor var_1673 = const()[name = tensor("op_1673"), val = tensor([3, 4, 12, 64])]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1656)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1673, x = q_21)[name = tensor("q")]; + tensor var_1675 = const()[name = tensor("op_1675"), val = tensor([3, 4, 12, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1663)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1675, x = k_21)[name = tensor("k")]; + tensor var_1677 = const()[name = tensor("op_1677"), val = tensor([3, 4, 12, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1670)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1677, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -1855,50 +1552,50 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_2108 = const()[name = tensor("op_2108"), val = tensor([2, 0, 1, 3])]; - tensor var_2113 = const()[name = tensor("op_2113"), val = tensor([36, 256])]; - tensor var_2109 = transpose(perm = var_2108, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_2113, x = var_2109)[name = tensor("attn_output_11")]; + tensor var_1680 = const()[name = tensor("op_1680"), val = tensor([2, 0, 1, 3])]; + tensor var_1685 = const()[name = tensor("op_1685"), val = tensor([36, 256])]; + tensor var_1681 = transpose(perm = var_1680, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1685, x = var_1681)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_2117 = const()[name = tensor("op_2117"), val = tensor([12, 3, 256])]; - tensor attn_output = reshape(shape = var_2117, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1689 = const()[name = tensor("op_1689"), val = tensor([12, 3, 256])]; + tensor attn_output = reshape(shape = var_1689, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; - tensor input_273 = add(x = query_5, y = sa2_out)[name = tensor("input_273")]; - tensor input_275_axes_0 = const()[name = tensor("input_275_axes_0"), val = tensor([-1])]; - tensor input_275 = layer_norm(axes = input_275_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1693, gamma = decoder_norm21_1_weight, x = input_273)[name = tensor("input_275")]; - tensor input_277 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_275)[name = tensor("linear_54")]; - tensor input_279 = relu(x = input_277)[name = tensor("input_279")]; - tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_279)[name = tensor("linear_55")]; - tensor input_281 = add(x = input_275, y = ffn_out)[name = tensor("input_281")]; + tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; + tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1264, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; + tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; + tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; + tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1693, gamma = decoder_norm22_1_weight, x = input_281)[name = tensor("xt")]; - tensor var_2137 = const()[name = tensor("op_2137"), val = tensor([1, 3, 12, 256])]; - tensor input = reshape(shape = var_2137, x = xt)[name = tensor("input")]; - tensor var_2139 = const()[name = tensor("op_2139"), val = tensor([-1])]; - tensor var_2140 = reduce_l2_norm(axes = var_2139, keep_dims = var_1696, x = input)[name = tensor("op_2140")]; - tensor const_50 = const()[name = tensor("const_50"), val = tensor(0x1.fffffep+127)]; - tensor clip_29 = clip(alpha = var_1687, beta = const_50, x = var_2140)[name = tensor("clip_29")]; - tensor var_2142 = real_div(x = input, y = clip_29)[name = tensor("op_2142")]; - tensor concat_17 = const()[name = tensor("concat_17"), val = tensor([3, 1, 256])]; - tensor reshape_0 = reshape(shape = concat_17, x = emb)[name = tensor("reshape_0")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1264, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1709 = const()[name = tensor("op_1709"), val = tensor([1, 3, 12, 256])]; + tensor input = reshape(shape = var_1709, x = xt)[name = tensor("input")]; + tensor var_1711 = const()[name = tensor("op_1711"), val = tensor([-1])]; + tensor var_1712 = reduce_l2_norm(axes = var_1711, keep_dims = var_1267, x = input)[name = tensor("op_1712")]; + tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; + tensor clip_29 = clip(alpha = var_1259, beta = const_42, x = var_1712)[name = tensor("clip_29")]; + tensor var_1714 = real_div(x = input, y = clip_29)[name = tensor("op_1714")]; + tensor concat_18 = const()[name = tensor("concat_18"), val = tensor([3, 1, 256])]; + tensor reshape_0 = reshape(shape = concat_18, x = emb)[name = tensor("reshape_0")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_18 = const()[name = tensor("concat_18"), val = tensor([3, 256, 12])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_2142)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_18, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_19 = const()[name = tensor("concat_19"), val = tensor([3, 256, 12])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1714)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_19, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = reshape_0, y = reshape_1)[name = tensor("matmul_2")]; - tensor concat_21 = const()[name = tensor("concat_21"), val = tensor([1, 3, 12])]; - tensor reshape_2 = reshape(shape = concat_21, x = matmul_2)[name = tensor("reshape_2")]; + tensor concat_22 = const()[name = tensor("concat_22"), val = tensor([1, 3, 12])]; + tensor reshape_2 = reshape(shape = concat_22, x = matmul_2)[name = tensor("reshape_2")]; tensor output_begin_0 = const()[name = tensor("output_begin_0"), val = tensor([0, 0, 1])]; tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 3, 11])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = reshape_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_2146")]; - tensor var_2148_axis_0 = const()[name = tensor("op_2148_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_2148_axis_0, values = (var_1844, nkv))[name = tensor("op_2148")]; - tensor var_2150_axis_0 = const()[name = tensor("op_2150_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_2150_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_2150")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1718")]; + tensor var_1720_axis_0 = const()[name = tensor("op_1720_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1720_axis_0, values = (var_1416, nkv))[name = tensor("op_1720")]; + tensor var_1722_axis_0 = const()[name = tensor("op_1722_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1722_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1722")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/DIHARD II/ls_eend_dih2_300ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/DIHARD II/ls_eend_dih2_300ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index e1c47545b7bdba75a868a59ea6e6ab6dc0d0e41a..8e095a48a721a063cdc467b823356c74e4f22086 100644 --- a/optimized/DIHARD II/ls_eend_dih2_300ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/DIHARD II/ls_eend_dih2_300ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6d583e00491898b933b00d04398308ad95254d3c0cb63fc12fc3ec98d0c88e9f -size 269882 +oid sha256:478d66dc60d867ab29487e24e4b0383a53bc514b7eb3a128278b457e65cf34be +size 222333 diff --git a/optimized/DIHARD II/ls_eend_dih2_300ms.mlpackage/Manifest.json b/optimized/DIHARD II/ls_eend_dih2_300ms.mlpackage/Manifest.json index 501f8579ae6f3448aa71e135e522e61bdefaafe0..88c216c22a4f9ead01fc2eadd5db5393572a42cb 100644 --- a/optimized/DIHARD II/ls_eend_dih2_300ms.mlpackage/Manifest.json +++ b/optimized/DIHARD II/ls_eend_dih2_300ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "D8E2269B-EBE8-4FF3-A320-0FCADEC1343F": { + "285FA0B6-5A0F-4A2B-A6A9-5B97AC37C032": { "author": "com.apple.CoreML", "description": "CoreML Model Weights", "name": "weights", "path": "com.apple.CoreML/weights" }, - "FD65A77B-98D8-4442-A27E-E38BCB9BE827": { + "721346E4-BD71-48C9-896A-FD7D8DF8170A": { "author": "com.apple.CoreML", "description": "CoreML Model Specification", "name": "model.mlmodel", "path": "com.apple.CoreML/model.mlmodel" } }, - "rootModelIdentifier": "FD65A77B-98D8-4442-A27E-E38BCB9BE827" + "rootModelIdentifier": "721346E4-BD71-48C9-896A-FD7D8DF8170A" } diff --git a/optimized/DIHARD II/ls_eend_dih2_400ms.mlmodelc/analytics/coremldata.bin b/optimized/DIHARD II/ls_eend_dih2_400ms.mlmodelc/analytics/coremldata.bin index bbbecc89564074ed8702417f0529ad258197401b..d73df7c54d5559cf59004784155dd2791e906d3a 100644 --- a/optimized/DIHARD II/ls_eend_dih2_400ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/DIHARD II/ls_eend_dih2_400ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5c3441417d312401803ad239d8f242af011bf8dd733b83cd79fbf5d7b28a1574 +oid sha256:e62282abfc6bca73e38e86971e6febdcb59a3039db237fcbd5b5772935158fd8 size 243 diff --git a/optimized/DIHARD II/ls_eend_dih2_400ms.mlmodelc/coremldata.bin b/optimized/DIHARD II/ls_eend_dih2_400ms.mlmodelc/coremldata.bin index c484b73aa35fa3d819bb3a88c6fafd98ba07cf6b..25bb06208279aad9e135d3b78eabe17e89fdb2e7 100644 --- a/optimized/DIHARD II/ls_eend_dih2_400ms.mlmodelc/coremldata.bin +++ b/optimized/DIHARD II/ls_eend_dih2_400ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:98eb81861e8948d14306386da0a0c59aff17b994b60512b38a64868ff049428f -size 1266 +oid sha256:3e771c47fb68b6679902d86013d0de2deecbcd09ce16f63eeab54867d494f4a7 +size 1357 diff --git a/optimized/DIHARD II/ls_eend_dih2_400ms.mlmodelc/metadata.json b/optimized/DIHARD II/ls_eend_dih2_400ms.mlmodelc/metadata.json index 66d2219496f7b0b166539f20176fd64300684aa9..f3776859eaf1fed9d8ee2c2c45636fa461598d5d 100644 --- a/optimized/DIHARD II/ls_eend_dih2_400ms.mlmodelc/metadata.json +++ b/optimized/DIHARD II/ls_eend_dih2_400ms.mlmodelc/metadata.json @@ -96,30 +96,30 @@ "Ios17.reshape" : 100, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 77, - "Split" : 16, + "Ios17.transpose" : 57, + "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 94, "Tile" : 17, - "Ios17.sliceByIndex" : 92, - "Ios16.sigmoid" : 17, + "Ios17.sliceByIndex" : 68, + "Ios16.sigmoid" : 5, "Ios16.reduceSum" : 1, "Ios17.squeeze" : 2, - "Ios17.layerNorm" : 45, - "Ios17.batchNorm" : 16, + "Ios17.layerNorm" : 33, + "Ios17.batchNorm" : 4, "Ios17.cast" : 32, "Ios17.sqrt" : 18, "Ios17.sub" : 32, - "Ios17.conv" : 49, + "Ios17.conv" : 13, "Ios17.realDiv" : 20, "Ios17.linear" : 56, "Ios17.concat" : 22, "Ios17.clip" : 38, "Ios17.greaterEqual" : 16, - "Ios16.silu" : 30, + "Ios16.silu" : 18, "Stack" : 6, "Ios16.cumsum" : 1, - "Ios17.mul" : 123 + "Ios17.mul" : 111 }, "computePrecision" : "Mixed (Float32, Int32)", "isUpdatable" : "0", @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"dih2\", \"model_label\": \"DIHARD II\", \"variant\": \"pipeline\", \"T\": 4, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"dih2\", \"model_label\": \"DIHARD II\", \"variant\": \"pipeline\", \"chunk_size\": 4, \"step_duration_ms\": 400, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/DIHARD II/ls_eend_dih2_400ms.mlmodelc/model.mil b/optimized/DIHARD II/ls_eend_dih2_400ms.mlmodelc/model.mil index ab2490e66dc769b71c76b7d12854b119a3d8d546..d04fbf3f7fb70564326a57bf225ae127afa5bbd7 100644 --- a/optimized/DIHARD II/ls_eend_dih2_400ms.mlmodelc/model.mil +++ b/optimized/DIHARD II/ls_eend_dih2_400ms.mlmodelc/model.mil @@ -217,6 +217,7 @@ program(1.0) tensor var_21 = const()[name = tensor("op_21"), val = tensor(0x0p+0)]; tensor var_24 = const()[name = tensor("op_24"), val = tensor(0x1.0c6f7ap-20)]; tensor var_27 = const()[name = tensor("op_27"), val = tensor(2)]; + tensor var_30 = const()[name = tensor("op_30"), val = tensor(0)]; tensor var_33 = const()[name = tensor("op_33"), val = tensor(1)]; tensor var_35 = const()[name = tensor("op_35"), val = tensor(0x1.4f8b58p-17)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(true)]; @@ -263,8 +264,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_77")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_78")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_196 = const()[name = tensor("op_196"), val = tensor([4, 1])]; tensor var_197 = reshape(shape = var_196, x = sqrt_s_t_1)[name = tensor("op_197")]; @@ -272,7 +273,7 @@ program(1.0) tensor var_199 = mul(x = qk_1, y = M_1)[name = tensor("op_199")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_76")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_199, y = v_1)[name = tensor("inner_1")]; tensor var_201_transpose_x_0 = const()[name = tensor("op_201_transpose_x_0"), val = tensor(false)]; tensor var_201_transpose_y_0 = const()[name = tensor("op_201_transpose_y_0"), val = tensor(false)]; @@ -293,7 +294,7 @@ program(1.0) tensor var_214 = real_div(x = new_kv_unnorm_1, y = var_213)[name = tensor("op_214")]; tensor var_215_perm_0 = const()[name = tensor("op_215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_75")]; + tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_24, x = var_215)[name = tensor("out_3")]; tensor var_219 = const()[name = tensor("op_219"), val = tensor([1, 4, 256])]; tensor out_5 = reshape(shape = var_219, x = out_3)[name = tensor("out_5")]; @@ -341,291 +342,176 @@ program(1.0) tensor var_252 = sub(x = var_17, y = use_roll_1)[name = tensor("op_252")]; tensor var_253 = mul(x = filled_window_1, y = var_252)[name = tensor("op_253")]; tensor var_254 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_254")]; - tensor input_21 = add(x = var_253, y = var_254)[name = tensor("input_21")]; + tensor window_3 = add(x = var_253, y = var_254)[name = tensor("window_3")]; tensor var_256 = const()[name = tensor("op_256"), val = tensor(0x1p+0)]; tensor var_257 = add(x = filled_1, y = var_256)[name = tensor("op_257")]; tensor clip_1 = clip(alpha = var_21, beta = var_16, x = var_257)[name = tensor("clip_1")]; + tensor var_260_begin_0 = const()[name = tensor("op_260_begin_0"), val = tensor([0, 1, 0])]; + tensor var_260_end_0 = const()[name = tensor("op_260_end_0"), val = tensor([1, 2, 256])]; + tensor var_260_end_mask_0 = const()[name = tensor("op_260_end_mask_0"), val = tensor([true, false, true])]; + tensor var_260 = slice_by_index(begin = var_260_begin_0, end = var_260_end_0, end_mask = var_260_end_mask_0, x = x_3)[name = tensor("op_260")]; + tensor var_263_begin_0 = const()[name = tensor("op_263_begin_0"), val = tensor([0, 1, 0])]; + tensor var_263_end_0 = const()[name = tensor("op_263_end_0"), val = tensor([1, 16, 256])]; + tensor var_263_end_mask_0 = const()[name = tensor("op_263_end_mask_0"), val = tensor([true, true, true])]; + tensor var_263 = slice_by_index(begin = var_263_begin_0, end = var_263_end_0, end_mask = var_263_end_mask_0, x = window_3)[name = tensor("op_263")]; + tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; + tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_263, var_260))[name = tensor("rolled_3")]; + tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; + tensor var_268 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_268")]; + tensor var_269_dtype_0 = const()[name = tensor("op_269_dtype_0"), val = tensor("fp32")]; + tensor var_270 = const()[name = tensor("op_270"), val = tensor([1, 16, 1])]; + tensor var_269 = cast(dtype = var_269_dtype_0, x = var_268)[name = tensor("cast_173")]; + tensor fill_mask_3 = reshape(shape = var_270, x = var_269)[name = tensor("fill_mask_3")]; + tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_260)[name = tensor("x_expanded_3")]; + tensor var_274 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_274")]; + tensor var_275 = mul(x = window_3, y = var_274)[name = tensor("op_275")]; + tensor var_276 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_276")]; + tensor filled_window_3 = add(x = var_275, y = var_276)[name = tensor("filled_window_3")]; + tensor var_278 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_278")]; + tensor var_279_dtype_0 = const()[name = tensor("op_279_dtype_0"), val = tensor("fp32")]; + tensor var_280 = const()[name = tensor("op_280"), val = tensor([1, 1, 1])]; + tensor var_279 = cast(dtype = var_279_dtype_0, x = var_278)[name = tensor("cast_172")]; + tensor use_roll_3 = reshape(shape = var_280, x = var_279)[name = tensor("use_roll_3")]; + tensor var_282 = sub(x = var_17, y = use_roll_3)[name = tensor("op_282")]; + tensor var_283 = mul(x = filled_window_3, y = var_282)[name = tensor("op_283")]; + tensor var_284 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_284")]; + tensor window_5 = add(x = var_283, y = var_284)[name = tensor("window_5")]; + tensor var_286 = const()[name = tensor("op_286"), val = tensor(0x1p+0)]; + tensor var_287 = add(x = clip_1, y = var_286)[name = tensor("op_287")]; + tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_287)[name = tensor("clip_3")]; + tensor var_290_begin_0 = const()[name = tensor("op_290_begin_0"), val = tensor([0, 2, 0])]; + tensor var_290_end_0 = const()[name = tensor("op_290_end_0"), val = tensor([1, 3, 256])]; + tensor var_290_end_mask_0 = const()[name = tensor("op_290_end_mask_0"), val = tensor([true, false, true])]; + tensor var_290 = slice_by_index(begin = var_290_begin_0, end = var_290_end_0, end_mask = var_290_end_mask_0, x = x_3)[name = tensor("op_290")]; + tensor var_293_begin_0 = const()[name = tensor("op_293_begin_0"), val = tensor([0, 1, 0])]; + tensor var_293_end_0 = const()[name = tensor("op_293_end_0"), val = tensor([1, 16, 256])]; + tensor var_293_end_mask_0 = const()[name = tensor("op_293_end_mask_0"), val = tensor([true, true, true])]; + tensor var_293 = slice_by_index(begin = var_293_begin_0, end = var_293_end_0, end_mask = var_293_end_mask_0, x = window_5)[name = tensor("op_293")]; + tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; + tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_293, var_290))[name = tensor("rolled_5")]; + tensor clip_4 = clip(alpha = var_21, beta = var_20, x = clip_3)[name = tensor("clip_4")]; + tensor var_298 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_298")]; + tensor var_299_dtype_0 = const()[name = tensor("op_299_dtype_0"), val = tensor("fp32")]; + tensor var_300 = const()[name = tensor("op_300"), val = tensor([1, 16, 1])]; + tensor var_299 = cast(dtype = var_299_dtype_0, x = var_298)[name = tensor("cast_171")]; + tensor fill_mask_5 = reshape(shape = var_300, x = var_299)[name = tensor("fill_mask_5")]; + tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_290)[name = tensor("x_expanded_5")]; + tensor var_304 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_304")]; + tensor var_305 = mul(x = window_5, y = var_304)[name = tensor("op_305")]; + tensor var_306 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_306")]; + tensor filled_window_5 = add(x = var_305, y = var_306)[name = tensor("filled_window_5")]; + tensor var_308 = greater_equal(x = clip_3, y = var_16)[name = tensor("op_308")]; + tensor var_309_dtype_0 = const()[name = tensor("op_309_dtype_0"), val = tensor("fp32")]; + tensor var_310 = const()[name = tensor("op_310"), val = tensor([1, 1, 1])]; + tensor var_309 = cast(dtype = var_309_dtype_0, x = var_308)[name = tensor("cast_170")]; + tensor use_roll_5 = reshape(shape = var_310, x = var_309)[name = tensor("use_roll_5")]; + tensor var_312 = sub(x = var_17, y = use_roll_5)[name = tensor("op_312")]; + tensor var_313 = mul(x = filled_window_5, y = var_312)[name = tensor("op_313")]; + tensor var_314 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_314")]; + tensor window_7 = add(x = var_313, y = var_314)[name = tensor("window_7")]; + tensor var_316 = const()[name = tensor("op_316"), val = tensor(0x1p+0)]; + tensor var_317 = add(x = clip_3, y = var_316)[name = tensor("op_317")]; + tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_317)[name = tensor("clip_5")]; + tensor var_320_begin_0 = const()[name = tensor("op_320_begin_0"), val = tensor([0, 3, 0])]; + tensor var_320_end_0 = const()[name = tensor("op_320_end_0"), val = tensor([1, 1, 256])]; + tensor var_320_end_mask_0 = const()[name = tensor("op_320_end_mask_0"), val = tensor([true, true, true])]; + tensor var_320 = slice_by_index(begin = var_320_begin_0, end = var_320_end_0, end_mask = var_320_end_mask_0, x = x_3)[name = tensor("op_320")]; + tensor var_323_begin_0 = const()[name = tensor("op_323_begin_0"), val = tensor([0, 1, 0])]; + tensor var_323_end_0 = const()[name = tensor("op_323_end_0"), val = tensor([1, 16, 256])]; + tensor var_323_end_mask_0 = const()[name = tensor("op_323_end_mask_0"), val = tensor([true, true, true])]; + tensor var_323 = slice_by_index(begin = var_323_begin_0, end = var_323_end_0, end_mask = var_323_end_mask_0, x = window_7)[name = tensor("op_323")]; + tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; + tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_323, var_320))[name = tensor("rolled_7")]; + tensor clip_6 = clip(alpha = var_21, beta = var_20, x = clip_5)[name = tensor("clip_6")]; + tensor var_328 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_328")]; + tensor var_329_dtype_0 = const()[name = tensor("op_329_dtype_0"), val = tensor("fp32")]; + tensor var_330 = const()[name = tensor("op_330"), val = tensor([1, 16, 1])]; + tensor var_329 = cast(dtype = var_329_dtype_0, x = var_328)[name = tensor("cast_169")]; + tensor fill_mask_7 = reshape(shape = var_330, x = var_329)[name = tensor("fill_mask_7")]; + tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_320)[name = tensor("x_expanded_7")]; + tensor var_334 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_334")]; + tensor var_335 = mul(x = window_7, y = var_334)[name = tensor("op_335")]; + tensor var_336 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_336")]; + tensor filled_window_7 = add(x = var_335, y = var_336)[name = tensor("filled_window_7")]; + tensor var_338 = greater_equal(x = clip_5, y = var_16)[name = tensor("op_338")]; + tensor var_339_dtype_0 = const()[name = tensor("op_339_dtype_0"), val = tensor("fp32")]; + tensor var_340 = const()[name = tensor("op_340"), val = tensor([1, 1, 1])]; + tensor var_339 = cast(dtype = var_339_dtype_0, x = var_338)[name = tensor("cast_168")]; + tensor use_roll_7 = reshape(shape = var_340, x = var_339)[name = tensor("use_roll_7")]; + tensor var_342 = sub(x = var_17, y = use_roll_7)[name = tensor("op_342")]; + tensor var_343 = mul(x = filled_window_7, y = var_342)[name = tensor("op_343")]; + tensor var_344 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_344")]; + tensor window_9 = add(x = var_343, y = var_344)[name = tensor("window_9")]; + tensor var_346 = const()[name = tensor("op_346"), val = tensor(0x1p+0)]; + tensor var_347 = add(x = clip_5, y = var_346)[name = tensor("op_347")]; + tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_347)[name = tensor("clip_7")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_30, interleave = input_21_interleave_0, values = (window_3, window_5, window_7, window_9))[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; - tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; + tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; tensor inputs_3_pad_type_0 = const()[name = tensor("inputs_3_pad_type_0"), val = tensor("valid")]; tensor inputs_3_strides_0 = const()[name = tensor("inputs_3_strides_0"), val = tensor([1])]; tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_74")]; - tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_278_split_sizes_0 = const()[name = tensor("op_278_split_sizes_0"), val = tensor([256, 256])]; - tensor var_278_axis_0 = const()[name = tensor("op_278_axis_0"), val = tensor(1)]; - tensor var_278_0, tensor var_278_1 = split(axis = var_278_axis_0, split_sizes = var_278_split_sizes_0, x = inputs_3)[name = tensor("op_278")]; - tensor var_280 = sigmoid(x = var_278_1)[name = tensor("op_280")]; - tensor inputs_5 = mul(x = var_278_0, y = var_280)[name = tensor("inputs_5")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; + tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; + tensor var_370_split_sizes_0 = const()[name = tensor("op_370_split_sizes_0"), val = tensor([256, 256])]; + tensor var_370_axis_0 = const()[name = tensor("op_370_axis_0"), val = tensor(1)]; + tensor var_370_0, tensor var_370_1 = split(axis = var_370_axis_0, split_sizes = var_370_split_sizes_0, x = inputs_3)[name = tensor("op_370")]; + tensor var_372 = sigmoid(x = var_370_1)[name = tensor("op_372")]; + tensor inputs_5 = mul(x = var_370_0, y = var_372)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; tensor outputs_aug_1_strides_0 = const()[name = tensor("outputs_aug_1_strides_0"), val = tensor([1])]; tensor outputs_aug_1_dilations_0 = const()[name = tensor("outputs_aug_1_dilations_0"), val = tensor([1])]; - tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; + tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; tensor input_25_begin_0 = const()[name = tensor("input_25_begin_0"), val = tensor([0, 0, 0])]; - tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([1, 256, 16])]; + tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([4, 256, 16])]; tensor input_25_end_mask_0 = const()[name = tensor("input_25_end_mask_0"), val = tensor([true, true, false])]; - tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; - tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; - tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; + tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; + tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; + tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; tensor input_29_pad_type_0 = const()[name = tensor("input_29_pad_type_0"), val = tensor("valid")]; tensor input_29_strides_0 = const()[name = tensor("input_29_strides_0"), val = tensor([1])]; tensor input_29_pad_0 = const()[name = tensor("input_29_pad_0"), val = tensor([0, 0])]; tensor input_29_dilations_0 = const()[name = tensor("input_29_dilations_0"), val = tensor([1])]; tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; - tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; + tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_311_begin_0 = const()[name = tensor("op_311_begin_0"), val = tensor([0, -1, 0])]; - tensor var_311_end_0 = const()[name = tensor("op_311_end_0"), val = tensor([1, 16, 256])]; - tensor var_311_end_mask_0 = const()[name = tensor("op_311_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_73")]; - tensor var_311 = slice_by_index(begin = var_311_begin_0, end = var_311_end_0, end_mask = var_311_end_mask_0, x = conv_out_1)[name = tensor("op_311")]; - tensor var_314_begin_0 = const()[name = tensor("op_314_begin_0"), val = tensor([0, 1, 0])]; - tensor var_314_end_0 = const()[name = tensor("op_314_end_0"), val = tensor([1, 2, 256])]; - tensor var_314_end_mask_0 = const()[name = tensor("op_314_end_mask_0"), val = tensor([true, false, true])]; - tensor var_314 = slice_by_index(begin = var_314_begin_0, end = var_314_end_0, end_mask = var_314_end_mask_0, x = x_3)[name = tensor("op_314")]; - tensor var_317_begin_0 = const()[name = tensor("op_317_begin_0"), val = tensor([0, 1, 0])]; - tensor var_317_end_0 = const()[name = tensor("op_317_end_0"), val = tensor([1, 16, 256])]; - tensor var_317_end_mask_0 = const()[name = tensor("op_317_end_mask_0"), val = tensor([true, true, true])]; - tensor var_317 = slice_by_index(begin = var_317_begin_0, end = var_317_end_0, end_mask = var_317_end_mask_0, x = input_21)[name = tensor("op_317")]; - tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_317, var_314))[name = tensor("rolled_3")]; - tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; - tensor var_322 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_322")]; - tensor var_323_dtype_0 = const()[name = tensor("op_323_dtype_0"), val = tensor("fp32")]; - tensor var_324 = const()[name = tensor("op_324"), val = tensor([1, 16, 1])]; - tensor var_323 = cast(dtype = var_323_dtype_0, x = var_322)[name = tensor("cast_173")]; - tensor fill_mask_3 = reshape(shape = var_324, x = var_323)[name = tensor("fill_mask_3")]; - tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_314)[name = tensor("x_expanded_3")]; - tensor var_328 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_328")]; - tensor var_329 = mul(x = input_21, y = var_328)[name = tensor("op_329")]; - tensor var_330 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_330")]; - tensor filled_window_3 = add(x = var_329, y = var_330)[name = tensor("filled_window_3")]; - tensor var_332 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_332")]; - tensor var_333_dtype_0 = const()[name = tensor("op_333_dtype_0"), val = tensor("fp32")]; - tensor var_334 = const()[name = tensor("op_334"), val = tensor([1, 1, 1])]; - tensor var_333 = cast(dtype = var_333_dtype_0, x = var_332)[name = tensor("cast_172")]; - tensor use_roll_3 = reshape(shape = var_334, x = var_333)[name = tensor("use_roll_3")]; - tensor var_336 = sub(x = var_17, y = use_roll_3)[name = tensor("op_336")]; - tensor var_337 = mul(x = filled_window_3, y = var_336)[name = tensor("op_337")]; - tensor var_338 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_338")]; - tensor input_31 = add(x = var_337, y = var_338)[name = tensor("input_31")]; - tensor var_340 = const()[name = tensor("op_340"), val = tensor(0x1p+0)]; - tensor var_341 = add(x = clip_1, y = var_340)[name = tensor("op_341")]; - tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_341)[name = tensor("clip_3")]; + tensor var_403_begin_0 = const()[name = tensor("op_403_begin_0"), val = tensor([0, -1, 0])]; + tensor var_403_end_0 = const()[name = tensor("op_403_end_0"), val = tensor([4, 16, 256])]; + tensor var_403_end_mask_0 = const()[name = tensor("op_403_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_403 = slice_by_index(begin = var_403_begin_0, end = var_403_end_0, end_mask = var_403_end_mask_0, x = conv_out_1)[name = tensor("op_403")]; + tensor var_405_perm_0 = const()[name = tensor("op_405_perm_0"), val = tensor([1, 0, 2])]; + tensor var_405 = transpose(perm = var_405_perm_0, x = var_403)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_405)[name = tensor("input_31")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; + tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; + tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; + tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; + tensor var_428 = const()[name = tensor("op_428"), val = tensor(0x1p-1)]; + tensor var_429 = mul(x = input_39, y = var_428)[name = tensor("op_429")]; + tensor input_41 = add(x = var_429, y = input_31)[name = tensor("input_41")]; + tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; + tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; + tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; + tensor input_45 = layer_norm(axes = input_45_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_43)[name = tensor("input_45")]; + tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; + tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; + tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; + tensor var_458 = const()[name = tensor("op_458"), val = tensor(0x1p-1)]; + tensor var_459 = mul(x = input_51, y = var_458)[name = tensor("op_459")]; + tensor input_53 = add(x = var_459, y = input_43)[name = tensor("input_53")]; tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; - tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_31)[name = tensor("x_7")]; - tensor input_33_perm_0 = const()[name = tensor("input_33_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_9_pad_type_0 = const()[name = tensor("inputs_9_pad_type_0"), val = tensor("valid")]; - tensor inputs_9_strides_0 = const()[name = tensor("inputs_9_strides_0"), val = tensor([1])]; - tensor inputs_9_pad_0 = const()[name = tensor("inputs_9_pad_0"), val = tensor([0, 0])]; - tensor inputs_9_dilations_0 = const()[name = tensor("inputs_9_dilations_0"), val = tensor([1])]; - tensor inputs_9_groups_0 = const()[name = tensor("inputs_9_groups_0"), val = tensor(1)]; - tensor input_33 = transpose(perm = input_33_perm_0, x = x_7)[name = tensor("transpose_72")]; - tensor inputs_9 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_9_dilations_0, groups = inputs_9_groups_0, pad = inputs_9_pad_0, pad_type = inputs_9_pad_type_0, strides = inputs_9_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_33)[name = tensor("inputs_9")]; - tensor var_362_split_sizes_0 = const()[name = tensor("op_362_split_sizes_0"), val = tensor([256, 256])]; - tensor var_362_axis_0 = const()[name = tensor("op_362_axis_0"), val = tensor(1)]; - tensor var_362_0, tensor var_362_1 = split(axis = var_362_axis_0, split_sizes = var_362_split_sizes_0, x = inputs_9)[name = tensor("op_362")]; - tensor var_364 = sigmoid(x = var_362_1)[name = tensor("op_364")]; - tensor inputs_11 = mul(x = var_362_0, y = var_364)[name = tensor("inputs_11")]; - tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; - tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; - tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; - tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_11)[name = tensor("outputs_aug_3")]; - tensor input_35_begin_0 = const()[name = tensor("input_35_begin_0"), val = tensor([0, 0, 0])]; - tensor input_35_end_0 = const()[name = tensor("input_35_end_0"), val = tensor([1, 256, 16])]; - tensor input_35_end_mask_0 = const()[name = tensor("input_35_end_mask_0"), val = tensor([true, true, false])]; - tensor input_35 = slice_by_index(begin = input_35_begin_0, end = input_35_end_0, end_mask = input_35_end_mask_0, x = outputs_aug_3)[name = tensor("input_35")]; - tensor inputs_13 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_35)[name = tensor("inputs_13")]; - tensor input_37 = silu(x = inputs_13)[name = tensor("input_37")]; - tensor input_39_pad_type_0 = const()[name = tensor("input_39_pad_type_0"), val = tensor("valid")]; - tensor input_39_strides_0 = const()[name = tensor("input_39_strides_0"), val = tensor([1])]; - tensor input_39_pad_0 = const()[name = tensor("input_39_pad_0"), val = tensor([0, 0])]; - tensor input_39_dilations_0 = const()[name = tensor("input_39_dilations_0"), val = tensor([1])]; - tensor input_39_groups_0 = const()[name = tensor("input_39_groups_0"), val = tensor(1)]; - tensor input_39 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_37)[name = tensor("input_39")]; - tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_395_begin_0 = const()[name = tensor("op_395_begin_0"), val = tensor([0, -1, 0])]; - tensor var_395_end_0 = const()[name = tensor("op_395_end_0"), val = tensor([1, 16, 256])]; - tensor var_395_end_mask_0 = const()[name = tensor("op_395_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_39)[name = tensor("transpose_71")]; - tensor var_395 = slice_by_index(begin = var_395_begin_0, end = var_395_end_0, end_mask = var_395_end_mask_0, x = conv_out_3)[name = tensor("op_395")]; - tensor var_398_begin_0 = const()[name = tensor("op_398_begin_0"), val = tensor([0, 2, 0])]; - tensor var_398_end_0 = const()[name = tensor("op_398_end_0"), val = tensor([1, 3, 256])]; - tensor var_398_end_mask_0 = const()[name = tensor("op_398_end_mask_0"), val = tensor([true, false, true])]; - tensor var_398 = slice_by_index(begin = var_398_begin_0, end = var_398_end_0, end_mask = var_398_end_mask_0, x = x_3)[name = tensor("op_398")]; - tensor var_401_begin_0 = const()[name = tensor("op_401_begin_0"), val = tensor([0, 1, 0])]; - tensor var_401_end_0 = const()[name = tensor("op_401_end_0"), val = tensor([1, 16, 256])]; - tensor var_401_end_mask_0 = const()[name = tensor("op_401_end_mask_0"), val = tensor([true, true, true])]; - tensor var_401 = slice_by_index(begin = var_401_begin_0, end = var_401_end_0, end_mask = var_401_end_mask_0, x = input_31)[name = tensor("op_401")]; - tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_401, var_398))[name = tensor("rolled_5")]; - tensor clip_4 = clip(alpha = var_21, beta = var_20, x = clip_3)[name = tensor("clip_4")]; - tensor var_406 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_406")]; - tensor var_407_dtype_0 = const()[name = tensor("op_407_dtype_0"), val = tensor("fp32")]; - tensor var_408 = const()[name = tensor("op_408"), val = tensor([1, 16, 1])]; - tensor var_407 = cast(dtype = var_407_dtype_0, x = var_406)[name = tensor("cast_171")]; - tensor fill_mask_5 = reshape(shape = var_408, x = var_407)[name = tensor("fill_mask_5")]; - tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_398)[name = tensor("x_expanded_5")]; - tensor var_412 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_412")]; - tensor var_413 = mul(x = input_31, y = var_412)[name = tensor("op_413")]; - tensor var_414 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_414")]; - tensor filled_window_5 = add(x = var_413, y = var_414)[name = tensor("filled_window_5")]; - tensor var_416 = greater_equal(x = clip_3, y = var_16)[name = tensor("op_416")]; - tensor var_417_dtype_0 = const()[name = tensor("op_417_dtype_0"), val = tensor("fp32")]; - tensor var_418 = const()[name = tensor("op_418"), val = tensor([1, 1, 1])]; - tensor var_417 = cast(dtype = var_417_dtype_0, x = var_416)[name = tensor("cast_170")]; - tensor use_roll_5 = reshape(shape = var_418, x = var_417)[name = tensor("use_roll_5")]; - tensor var_420 = sub(x = var_17, y = use_roll_5)[name = tensor("op_420")]; - tensor var_421 = mul(x = filled_window_5, y = var_420)[name = tensor("op_421")]; - tensor var_422 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_422")]; - tensor input_41 = add(x = var_421, y = var_422)[name = tensor("input_41")]; - tensor var_424 = const()[name = tensor("op_424"), val = tensor(0x1p+0)]; - tensor var_425 = add(x = clip_3, y = var_424)[name = tensor("op_425")]; - tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_425)[name = tensor("clip_5")]; - tensor x_9_axes_0 = const()[name = tensor("x_9_axes_0"), val = tensor([-1])]; - tensor x_9 = layer_norm(axes = x_9_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_41)[name = tensor("x_9")]; - tensor input_43_perm_0 = const()[name = tensor("input_43_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_15_pad_type_0 = const()[name = tensor("inputs_15_pad_type_0"), val = tensor("valid")]; - tensor inputs_15_strides_0 = const()[name = tensor("inputs_15_strides_0"), val = tensor([1])]; - tensor inputs_15_pad_0 = const()[name = tensor("inputs_15_pad_0"), val = tensor([0, 0])]; - tensor inputs_15_dilations_0 = const()[name = tensor("inputs_15_dilations_0"), val = tensor([1])]; - tensor inputs_15_groups_0 = const()[name = tensor("inputs_15_groups_0"), val = tensor(1)]; - tensor input_43 = transpose(perm = input_43_perm_0, x = x_9)[name = tensor("transpose_70")]; - tensor inputs_15 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_15_dilations_0, groups = inputs_15_groups_0, pad = inputs_15_pad_0, pad_type = inputs_15_pad_type_0, strides = inputs_15_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_43)[name = tensor("inputs_15")]; - tensor var_446_split_sizes_0 = const()[name = tensor("op_446_split_sizes_0"), val = tensor([256, 256])]; - tensor var_446_axis_0 = const()[name = tensor("op_446_axis_0"), val = tensor(1)]; - tensor var_446_0, tensor var_446_1 = split(axis = var_446_axis_0, split_sizes = var_446_split_sizes_0, x = inputs_15)[name = tensor("op_446")]; - tensor var_448 = sigmoid(x = var_446_1)[name = tensor("op_448")]; - tensor inputs_17 = mul(x = var_446_0, y = var_448)[name = tensor("inputs_17")]; - tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; - tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; - tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; - tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_17)[name = tensor("outputs_aug_5")]; - tensor input_45_begin_0 = const()[name = tensor("input_45_begin_0"), val = tensor([0, 0, 0])]; - tensor input_45_end_0 = const()[name = tensor("input_45_end_0"), val = tensor([1, 256, 16])]; - tensor input_45_end_mask_0 = const()[name = tensor("input_45_end_mask_0"), val = tensor([true, true, false])]; - tensor input_45 = slice_by_index(begin = input_45_begin_0, end = input_45_end_0, end_mask = input_45_end_mask_0, x = outputs_aug_5)[name = tensor("input_45")]; - tensor inputs_19 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_45)[name = tensor("inputs_19")]; - tensor input_47 = silu(x = inputs_19)[name = tensor("input_47")]; - tensor input_49_pad_type_0 = const()[name = tensor("input_49_pad_type_0"), val = tensor("valid")]; - tensor input_49_strides_0 = const()[name = tensor("input_49_strides_0"), val = tensor([1])]; - tensor input_49_pad_0 = const()[name = tensor("input_49_pad_0"), val = tensor([0, 0])]; - tensor input_49_dilations_0 = const()[name = tensor("input_49_dilations_0"), val = tensor([1])]; - tensor input_49_groups_0 = const()[name = tensor("input_49_groups_0"), val = tensor(1)]; - tensor input_49 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_49_dilations_0, groups = input_49_groups_0, pad = input_49_pad_0, pad_type = input_49_pad_type_0, strides = input_49_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_47)[name = tensor("input_49")]; - tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_479_begin_0 = const()[name = tensor("op_479_begin_0"), val = tensor([0, -1, 0])]; - tensor var_479_end_0 = const()[name = tensor("op_479_end_0"), val = tensor([1, 16, 256])]; - tensor var_479_end_mask_0 = const()[name = tensor("op_479_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_49)[name = tensor("transpose_69")]; - tensor var_479 = slice_by_index(begin = var_479_begin_0, end = var_479_end_0, end_mask = var_479_end_mask_0, x = conv_out_5)[name = tensor("op_479")]; - tensor var_482_begin_0 = const()[name = tensor("op_482_begin_0"), val = tensor([0, 3, 0])]; - tensor var_482_end_0 = const()[name = tensor("op_482_end_0"), val = tensor([1, 1, 256])]; - tensor var_482_end_mask_0 = const()[name = tensor("op_482_end_mask_0"), val = tensor([true, true, true])]; - tensor var_482 = slice_by_index(begin = var_482_begin_0, end = var_482_end_0, end_mask = var_482_end_mask_0, x = x_3)[name = tensor("op_482")]; - tensor var_485_begin_0 = const()[name = tensor("op_485_begin_0"), val = tensor([0, 1, 0])]; - tensor var_485_end_0 = const()[name = tensor("op_485_end_0"), val = tensor([1, 16, 256])]; - tensor var_485_end_mask_0 = const()[name = tensor("op_485_end_mask_0"), val = tensor([true, true, true])]; - tensor var_485 = slice_by_index(begin = var_485_begin_0, end = var_485_end_0, end_mask = var_485_end_mask_0, x = input_41)[name = tensor("op_485")]; - tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; - tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_485, var_482))[name = tensor("rolled_7")]; - tensor clip_6 = clip(alpha = var_21, beta = var_20, x = clip_5)[name = tensor("clip_6")]; - tensor var_490 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_490")]; - tensor var_491_dtype_0 = const()[name = tensor("op_491_dtype_0"), val = tensor("fp32")]; - tensor var_492 = const()[name = tensor("op_492"), val = tensor([1, 16, 1])]; - tensor var_491 = cast(dtype = var_491_dtype_0, x = var_490)[name = tensor("cast_169")]; - tensor fill_mask_7 = reshape(shape = var_492, x = var_491)[name = tensor("fill_mask_7")]; - tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_482)[name = tensor("x_expanded_7")]; - tensor var_496 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_496")]; - tensor var_497 = mul(x = input_41, y = var_496)[name = tensor("op_497")]; - tensor var_498 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_498")]; - tensor filled_window_7 = add(x = var_497, y = var_498)[name = tensor("filled_window_7")]; - tensor var_500 = greater_equal(x = clip_5, y = var_16)[name = tensor("op_500")]; - tensor var_501_dtype_0 = const()[name = tensor("op_501_dtype_0"), val = tensor("fp32")]; - tensor var_502 = const()[name = tensor("op_502"), val = tensor([1, 1, 1])]; - tensor var_501 = cast(dtype = var_501_dtype_0, x = var_500)[name = tensor("cast_168")]; - tensor use_roll_7 = reshape(shape = var_502, x = var_501)[name = tensor("use_roll_7")]; - tensor var_504 = sub(x = var_17, y = use_roll_7)[name = tensor("op_504")]; - tensor var_505 = mul(x = filled_window_7, y = var_504)[name = tensor("op_505")]; - tensor var_506 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_506")]; - tensor input_51 = add(x = var_505, y = var_506)[name = tensor("input_51")]; - tensor var_508 = const()[name = tensor("op_508"), val = tensor(0x1p+0)]; - tensor var_509 = add(x = clip_5, y = var_508)[name = tensor("op_509")]; - tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_509)[name = tensor("clip_7")]; - tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; - tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_51)[name = tensor("x_11")]; - tensor input_53_perm_0 = const()[name = tensor("input_53_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_21_pad_type_0 = const()[name = tensor("inputs_21_pad_type_0"), val = tensor("valid")]; - tensor inputs_21_strides_0 = const()[name = tensor("inputs_21_strides_0"), val = tensor([1])]; - tensor inputs_21_pad_0 = const()[name = tensor("inputs_21_pad_0"), val = tensor([0, 0])]; - tensor inputs_21_dilations_0 = const()[name = tensor("inputs_21_dilations_0"), val = tensor([1])]; - tensor inputs_21_groups_0 = const()[name = tensor("inputs_21_groups_0"), val = tensor(1)]; - tensor input_53 = transpose(perm = input_53_perm_0, x = x_11)[name = tensor("transpose_68")]; - tensor inputs_21 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_21_dilations_0, groups = inputs_21_groups_0, pad = inputs_21_pad_0, pad_type = inputs_21_pad_type_0, strides = inputs_21_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_53)[name = tensor("inputs_21")]; - tensor var_530_split_sizes_0 = const()[name = tensor("op_530_split_sizes_0"), val = tensor([256, 256])]; - tensor var_530_axis_0 = const()[name = tensor("op_530_axis_0"), val = tensor(1)]; - tensor var_530_0, tensor var_530_1 = split(axis = var_530_axis_0, split_sizes = var_530_split_sizes_0, x = inputs_21)[name = tensor("op_530")]; - tensor var_532 = sigmoid(x = var_530_1)[name = tensor("op_532")]; - tensor inputs_23 = mul(x = var_530_0, y = var_532)[name = tensor("inputs_23")]; - tensor outputs_aug_7_pad_type_0 = const()[name = tensor("outputs_aug_7_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_7_pad_0 = const()[name = tensor("outputs_aug_7_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_7_groups_0 = const()[name = tensor("outputs_aug_7_groups_0"), val = tensor(256)]; - tensor outputs_aug_7_strides_0 = const()[name = tensor("outputs_aug_7_strides_0"), val = tensor([1])]; - tensor outputs_aug_7_dilations_0 = const()[name = tensor("outputs_aug_7_dilations_0"), val = tensor([1])]; - tensor outputs_aug_7 = conv(dilations = outputs_aug_7_dilations_0, groups = outputs_aug_7_groups_0, pad = outputs_aug_7_pad_0, pad_type = outputs_aug_7_pad_type_0, strides = outputs_aug_7_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_23)[name = tensor("outputs_aug_7")]; - tensor input_55_begin_0 = const()[name = tensor("input_55_begin_0"), val = tensor([0, 0, 0])]; - tensor input_55_end_0 = const()[name = tensor("input_55_end_0"), val = tensor([1, 256, 16])]; - tensor input_55_end_mask_0 = const()[name = tensor("input_55_end_mask_0"), val = tensor([true, true, false])]; - tensor input_55 = slice_by_index(begin = input_55_begin_0, end = input_55_end_0, end_mask = input_55_end_mask_0, x = outputs_aug_7)[name = tensor("input_55")]; - tensor inputs_25 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_55)[name = tensor("inputs_25")]; - tensor input_57 = silu(x = inputs_25)[name = tensor("input_57")]; - tensor input_59_pad_type_0 = const()[name = tensor("input_59_pad_type_0"), val = tensor("valid")]; - tensor input_59_strides_0 = const()[name = tensor("input_59_strides_0"), val = tensor([1])]; - tensor input_59_pad_0 = const()[name = tensor("input_59_pad_0"), val = tensor([0, 0])]; - tensor input_59_dilations_0 = const()[name = tensor("input_59_dilations_0"), val = tensor([1])]; - tensor input_59_groups_0 = const()[name = tensor("input_59_groups_0"), val = tensor(1)]; - tensor input_59 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_59_dilations_0, groups = input_59_groups_0, pad = input_59_pad_0, pad_type = input_59_pad_type_0, strides = input_59_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_57)[name = tensor("input_59")]; - tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_563_begin_0 = const()[name = tensor("op_563_begin_0"), val = tensor([0, -1, 0])]; - tensor var_563_end_0 = const()[name = tensor("op_563_end_0"), val = tensor([1, 16, 256])]; - tensor var_563_end_mask_0 = const()[name = tensor("op_563_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_59)[name = tensor("transpose_67")]; - tensor var_563 = slice_by_index(begin = var_563_begin_0, end = var_563_end_0, end_mask = var_563_end_mask_0, x = conv_out_7)[name = tensor("op_563")]; - tensor var_566_interleave_0 = const()[name = tensor("op_566_interleave_0"), val = tensor(false)]; - tensor var_566 = concat(axis = var_33, interleave = var_566_interleave_0, values = (var_311, var_395, var_479, var_563))[name = tensor("op_566")]; - tensor input_61 = add(x = x_3, y = var_566)[name = tensor("input_61")]; - tensor input_63_axes_0 = const()[name = tensor("input_63_axes_0"), val = tensor([-1])]; - tensor input_63 = layer_norm(axes = input_63_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_61)[name = tensor("input_63")]; - tensor inputs_27 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_63)[name = tensor("linear_8")]; - tensor input_65 = silu(x = inputs_27)[name = tensor("input_65")]; - tensor input_69 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_65)[name = tensor("linear_9")]; - tensor var_589 = const()[name = tensor("op_589"), val = tensor(0x1p-1)]; - tensor var_590 = mul(x = input_69, y = var_589)[name = tensor("op_590")]; - tensor input_71 = add(x = var_590, y = input_61)[name = tensor("input_71")]; - tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; - tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_71)[name = tensor("input_73")]; - tensor input_75_axes_0 = const()[name = tensor("input_75_axes_0"), val = tensor([-1])]; - tensor input_75 = layer_norm(axes = input_75_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_73)[name = tensor("input_75")]; - tensor inputs_29 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_75)[name = tensor("linear_10")]; - tensor input_77 = silu(x = inputs_29)[name = tensor("input_77")]; - tensor input_81 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_77)[name = tensor("linear_11")]; - tensor var_619 = const()[name = tensor("op_619"), val = tensor(0x1p-1)]; - tensor var_620 = mul(x = input_81, y = var_619)[name = tensor("op_620")]; - tensor input_83 = add(x = var_620, y = input_73)[name = tensor("input_83")]; - tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; - tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_83)[name = tensor("x_13")]; + tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_3_end_0 = const()[name = tensor("prev_kv_3_end_0"), val = tensor([2, 1, 4, 64, 64])]; tensor prev_kv_3_end_mask_0 = const()[name = tensor("prev_kv_3_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -636,389 +522,274 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_634 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_13)[name = tensor("linear_12")]; - tensor var_635 = const()[name = tensor("op_635"), val = tensor([1, 4, 4, 64])]; - tensor var_636 = reshape(shape = var_635, x = var_634)[name = tensor("op_636")]; + tensor var_473 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_474 = const()[name = tensor("op_474"), val = tensor([1, 4, 4, 64])]; + tensor var_475 = reshape(shape = var_474, x = var_473)[name = tensor("op_475")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_640 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_13)[name = tensor("linear_13")]; - tensor var_641 = const()[name = tensor("op_641"), val = tensor(0x1p-3)]; - tensor var_642 = mul(x = var_640, y = var_641)[name = tensor("op_642")]; - tensor var_643 = const()[name = tensor("op_643"), val = tensor([1, 4, 4, 64])]; - tensor var_644 = reshape(shape = var_643, x = var_642)[name = tensor("op_644")]; + tensor var_479 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_480 = const()[name = tensor("op_480"), val = tensor(0x1p-3)]; + tensor var_481 = mul(x = var_479, y = var_480)[name = tensor("op_481")]; + tensor var_482 = const()[name = tensor("op_482"), val = tensor([1, 4, 4, 64])]; + tensor var_483 = reshape(shape = var_482, x = var_481)[name = tensor("op_483")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_648 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_13)[name = tensor("linear_14")]; - tensor var_649 = const()[name = tensor("op_649"), val = tensor([1, 4, 4, 64])]; - tensor var_650 = reshape(shape = var_649, x = var_648)[name = tensor("op_650")]; + tensor var_487 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_488 = const()[name = tensor("op_488"), val = tensor([1, 4, 4, 64])]; + tensor var_489 = reshape(shape = var_488, x = var_487)[name = tensor("op_489")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_87 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_13)[name = tensor("linear_15")]; + tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; tensor s_t_3 = add(x = prev_scale_3, y = encoder__t_index)[name = tensor("s_t_3")]; tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_644)[name = tensor("transpose_65")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_636)[name = tensor("transpose_66")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_483)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_475)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_660 = const()[name = tensor("op_660"), val = tensor([4, 1])]; - tensor var_661 = reshape(shape = var_660, x = sqrt_s_t_3)[name = tensor("op_661")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_661)[name = tensor("M_3")]; - tensor var_663 = mul(x = qk_3, y = M_3)[name = tensor("op_663")]; + tensor var_499 = const()[name = tensor("op_499"), val = tensor([4, 1])]; + tensor var_500 = reshape(shape = var_499, x = sqrt_s_t_3)[name = tensor("op_500")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_500)[name = tensor("M_3")]; + tensor var_502 = mul(x = qk_3, y = M_3)[name = tensor("op_502")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_650)[name = tensor("transpose_64")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_663, y = v_3)[name = tensor("inner_3")]; - tensor var_665_transpose_x_0 = const()[name = tensor("op_665_transpose_x_0"), val = tensor(false)]; - tensor var_665_transpose_y_0 = const()[name = tensor("op_665_transpose_y_0"), val = tensor(false)]; - tensor var_665 = matmul(transpose_x = var_665_transpose_x_0, transpose_y = var_665_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_665")]; - tensor var_666 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_666")]; - tensor var_667 = const()[name = tensor("op_667"), val = tensor([1, 1, 4, 1])]; - tensor var_668 = reshape(shape = var_667, x = var_666)[name = tensor("op_668")]; - tensor cross_3 = mul(x = var_665, y = var_668)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_489)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_502, y = v_3)[name = tensor("inner_3")]; + tensor var_504_transpose_x_0 = const()[name = tensor("op_504_transpose_x_0"), val = tensor(false)]; + tensor var_504_transpose_y_0 = const()[name = tensor("op_504_transpose_y_0"), val = tensor(false)]; + tensor var_504 = matmul(transpose_x = var_504_transpose_x_0, transpose_y = var_504_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_504")]; + tensor var_505 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_505")]; + tensor var_506 = const()[name = tensor("op_506"), val = tensor([1, 1, 4, 1])]; + tensor var_507 = reshape(shape = var_506, x = var_505)[name = tensor("op_507")]; + tensor cross_3 = mul(x = var_504, y = var_507)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_671 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_671")]; - tensor var_673_transpose_x_1 = const()[name = tensor("op_673_transpose_x_1"), val = tensor(true)]; - tensor var_673_transpose_y_1 = const()[name = tensor("op_673_transpose_y_1"), val = tensor(false)]; - tensor var_673 = matmul(transpose_x = var_673_transpose_x_1, transpose_y = var_673_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_673")]; - tensor new_kv_unnorm_3 = add(x = var_671, y = var_673)[name = tensor("new_kv_unnorm_3")]; - tensor var_675 = const()[name = tensor("op_675"), val = tensor(0x1p+2)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_675)[name = tensor("new_scale_3")]; - tensor var_677 = sqrt(x = new_scale_3)[name = tensor("op_677")]; - tensor var_678 = real_div(x = new_kv_unnorm_3, y = var_677)[name = tensor("op_678")]; - tensor var_679_perm_0 = const()[name = tensor("op_679_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_510 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_510")]; + tensor var_512_transpose_x_1 = const()[name = tensor("op_512_transpose_x_1"), val = tensor(true)]; + tensor var_512_transpose_y_1 = const()[name = tensor("op_512_transpose_y_1"), val = tensor(false)]; + tensor var_512 = matmul(transpose_x = var_512_transpose_x_1, transpose_y = var_512_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_512")]; + tensor new_kv_unnorm_3 = add(x = var_510, y = var_512)[name = tensor("new_kv_unnorm_3")]; + tensor var_514 = const()[name = tensor("op_514"), val = tensor(0x1p+2)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_514)[name = tensor("new_scale_3")]; + tensor var_516 = sqrt(x = new_scale_3)[name = tensor("op_516")]; + tensor var_517 = real_div(x = new_kv_unnorm_3, y = var_516)[name = tensor("op_517")]; + tensor var_518_perm_0 = const()[name = tensor("op_518_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_679 = transpose(perm = var_679_perm_0, x = out_7)[name = tensor("transpose_63")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_679)[name = tensor("out_9")]; - tensor var_683 = const()[name = tensor("op_683"), val = tensor([1, 4, 256])]; - tensor out_11 = reshape(shape = var_683, x = out_9)[name = tensor("out_11")]; - tensor var_685 = silu(x = input_87)[name = tensor("op_685")]; - tensor input_89 = mul(x = var_685, y = out_11)[name = tensor("input_89")]; - tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_89)[name = tensor("linear_16")]; - tensor x_15 = add(x = input_83, y = ret_out_3)[name = tensor("x_15")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor var_518 = transpose(perm = var_518_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_518)[name = tensor("out_9")]; + tensor var_522 = const()[name = tensor("op_522"), val = tensor([1, 4, 256])]; + tensor out_11 = reshape(shape = var_522, x = out_9)[name = tensor("out_11")]; + tensor var_524 = silu(x = input_57)[name = tensor("op_524")]; + tensor input_59 = mul(x = var_524, y = out_11)[name = tensor("input_59")]; + tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; + tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; + tensor window_11_begin_0 = const()[name = tensor("window_11_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_11_end_0 = const()[name = tensor("window_11_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_11_end_mask_0 = const()[name = tensor("window_11_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_11_squeeze_mask_0 = const()[name = tensor("window_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_11 = slice_by_index(begin = window_11_begin_0, end = window_11_end_0, end_mask = window_11_end_mask_0, squeeze_mask = window_11_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_11")]; tensor filled_9_begin_0 = const()[name = tensor("filled_9_begin_0"), val = tensor([1, 0])]; tensor filled_9_end_0 = const()[name = tensor("filled_9_end_0"), val = tensor([2, 1])]; tensor filled_9_end_mask_0 = const()[name = tensor("filled_9_end_mask_0"), val = tensor([false, true])]; tensor filled_9_squeeze_mask_0 = const()[name = tensor("filled_9_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_9 = slice_by_index(begin = filled_9_begin_0, end = filled_9_end_0, end_mask = filled_9_end_mask_0, squeeze_mask = filled_9_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_9")]; - tensor var_694_begin_0 = const()[name = tensor("op_694_begin_0"), val = tensor([0, 0, 0])]; - tensor var_694_end_0 = const()[name = tensor("op_694_end_0"), val = tensor([1, 1, 256])]; - tensor var_694_end_mask_0 = const()[name = tensor("op_694_end_mask_0"), val = tensor([true, false, true])]; - tensor var_694 = slice_by_index(begin = var_694_begin_0, end = var_694_end_0, end_mask = var_694_end_mask_0, x = x_15)[name = tensor("op_694")]; - tensor var_697_begin_0 = const()[name = tensor("op_697_begin_0"), val = tensor([0, 1, 0])]; - tensor var_697_end_0 = const()[name = tensor("op_697_end_0"), val = tensor([1, 16, 256])]; - tensor var_697_end_mask_0 = const()[name = tensor("op_697_end_mask_0"), val = tensor([true, true, true])]; - tensor var_697 = slice_by_index(begin = var_697_begin_0, end = var_697_end_0, end_mask = var_697_end_mask_0, x = window_3)[name = tensor("op_697")]; + tensor var_533_begin_0 = const()[name = tensor("op_533_begin_0"), val = tensor([0, 0, 0])]; + tensor var_533_end_0 = const()[name = tensor("op_533_end_0"), val = tensor([1, 1, 256])]; + tensor var_533_end_mask_0 = const()[name = tensor("op_533_end_mask_0"), val = tensor([true, false, true])]; + tensor var_533 = slice_by_index(begin = var_533_begin_0, end = var_533_end_0, end_mask = var_533_end_mask_0, x = x_9)[name = tensor("op_533")]; + tensor var_536_begin_0 = const()[name = tensor("op_536_begin_0"), val = tensor([0, 1, 0])]; + tensor var_536_end_0 = const()[name = tensor("op_536_end_0"), val = tensor([1, 16, 256])]; + tensor var_536_end_mask_0 = const()[name = tensor("op_536_end_mask_0"), val = tensor([true, true, true])]; + tensor var_536 = slice_by_index(begin = var_536_begin_0, end = var_536_end_0, end_mask = var_536_end_mask_0, x = window_11)[name = tensor("op_536")]; tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; - tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_697, var_694))[name = tensor("rolled_9")]; + tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_536, var_533))[name = tensor("rolled_9")]; tensor clip_8 = clip(alpha = var_21, beta = var_20, x = filled_9)[name = tensor("clip_8")]; - tensor var_702 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_702")]; - tensor var_703_dtype_0 = const()[name = tensor("op_703_dtype_0"), val = tensor("fp32")]; - tensor var_704 = const()[name = tensor("op_704"), val = tensor([1, 16, 1])]; - tensor var_703 = cast(dtype = var_703_dtype_0, x = var_702)[name = tensor("cast_167")]; - tensor fill_mask_9 = reshape(shape = var_704, x = var_703)[name = tensor("fill_mask_9")]; + tensor var_541 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_541")]; + tensor var_542_dtype_0 = const()[name = tensor("op_542_dtype_0"), val = tensor("fp32")]; + tensor var_543 = const()[name = tensor("op_543"), val = tensor([1, 16, 1])]; + tensor var_542 = cast(dtype = var_542_dtype_0, x = var_541)[name = tensor("cast_167")]; + tensor fill_mask_9 = reshape(shape = var_543, x = var_542)[name = tensor("fill_mask_9")]; tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_694)[name = tensor("x_expanded_9")]; - tensor var_708 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_708")]; - tensor var_709 = mul(x = window_3, y = var_708)[name = tensor("op_709")]; - tensor var_710 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_710")]; - tensor filled_window_9 = add(x = var_709, y = var_710)[name = tensor("filled_window_9")]; - tensor var_712 = greater_equal(x = filled_9, y = var_16)[name = tensor("op_712")]; - tensor var_713_dtype_0 = const()[name = tensor("op_713_dtype_0"), val = tensor("fp32")]; - tensor var_714 = const()[name = tensor("op_714"), val = tensor([1, 1, 1])]; - tensor var_713 = cast(dtype = var_713_dtype_0, x = var_712)[name = tensor("cast_166")]; - tensor use_roll_9 = reshape(shape = var_714, x = var_713)[name = tensor("use_roll_9")]; - tensor var_716 = sub(x = var_17, y = use_roll_9)[name = tensor("op_716")]; - tensor var_717 = mul(x = filled_window_9, y = var_716)[name = tensor("op_717")]; - tensor var_718 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_718")]; - tensor input_91 = add(x = var_717, y = var_718)[name = tensor("input_91")]; - tensor var_720 = const()[name = tensor("op_720"), val = tensor(0x1p+0)]; - tensor var_721 = add(x = filled_9, y = var_720)[name = tensor("op_721")]; - tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_721)[name = tensor("clip_9")]; - tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; - tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_91)[name = tensor("x_17")]; - tensor input_93_perm_0 = const()[name = tensor("input_93_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_31_pad_type_0 = const()[name = tensor("inputs_31_pad_type_0"), val = tensor("valid")]; - tensor inputs_31_strides_0 = const()[name = tensor("inputs_31_strides_0"), val = tensor([1])]; - tensor inputs_31_pad_0 = const()[name = tensor("inputs_31_pad_0"), val = tensor([0, 0])]; - tensor inputs_31_dilations_0 = const()[name = tensor("inputs_31_dilations_0"), val = tensor([1])]; - tensor inputs_31_groups_0 = const()[name = tensor("inputs_31_groups_0"), val = tensor(1)]; - tensor input_93 = transpose(perm = input_93_perm_0, x = x_17)[name = tensor("transpose_62")]; - tensor inputs_31 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_31_dilations_0, groups = inputs_31_groups_0, pad = inputs_31_pad_0, pad_type = inputs_31_pad_type_0, strides = inputs_31_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_93)[name = tensor("inputs_31")]; - tensor var_742_split_sizes_0 = const()[name = tensor("op_742_split_sizes_0"), val = tensor([256, 256])]; - tensor var_742_axis_0 = const()[name = tensor("op_742_axis_0"), val = tensor(1)]; - tensor var_742_0, tensor var_742_1 = split(axis = var_742_axis_0, split_sizes = var_742_split_sizes_0, x = inputs_31)[name = tensor("op_742")]; - tensor var_744 = sigmoid(x = var_742_1)[name = tensor("op_744")]; - tensor inputs_33 = mul(x = var_742_0, y = var_744)[name = tensor("inputs_33")]; - tensor outputs_aug_9_pad_type_0 = const()[name = tensor("outputs_aug_9_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_9_pad_0 = const()[name = tensor("outputs_aug_9_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_9_groups_0 = const()[name = tensor("outputs_aug_9_groups_0"), val = tensor(256)]; - tensor outputs_aug_9_strides_0 = const()[name = tensor("outputs_aug_9_strides_0"), val = tensor([1])]; - tensor outputs_aug_9_dilations_0 = const()[name = tensor("outputs_aug_9_dilations_0"), val = tensor([1])]; - tensor outputs_aug_9 = conv(dilations = outputs_aug_9_dilations_0, groups = outputs_aug_9_groups_0, pad = outputs_aug_9_pad_0, pad_type = outputs_aug_9_pad_type_0, strides = outputs_aug_9_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_33)[name = tensor("outputs_aug_9")]; - tensor input_95_begin_0 = const()[name = tensor("input_95_begin_0"), val = tensor([0, 0, 0])]; - tensor input_95_end_0 = const()[name = tensor("input_95_end_0"), val = tensor([1, 256, 16])]; - tensor input_95_end_mask_0 = const()[name = tensor("input_95_end_mask_0"), val = tensor([true, true, false])]; - tensor input_95 = slice_by_index(begin = input_95_begin_0, end = input_95_end_0, end_mask = input_95_end_mask_0, x = outputs_aug_9)[name = tensor("input_95")]; - tensor inputs_35 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_95)[name = tensor("inputs_35")]; - tensor input_97 = silu(x = inputs_35)[name = tensor("input_97")]; - tensor input_99_pad_type_0 = const()[name = tensor("input_99_pad_type_0"), val = tensor("valid")]; - tensor input_99_strides_0 = const()[name = tensor("input_99_strides_0"), val = tensor([1])]; - tensor input_99_pad_0 = const()[name = tensor("input_99_pad_0"), val = tensor([0, 0])]; - tensor input_99_dilations_0 = const()[name = tensor("input_99_dilations_0"), val = tensor([1])]; - tensor input_99_groups_0 = const()[name = tensor("input_99_groups_0"), val = tensor(1)]; - tensor input_99 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_99_dilations_0, groups = input_99_groups_0, pad = input_99_pad_0, pad_type = input_99_pad_type_0, strides = input_99_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_97)[name = tensor("input_99")]; - tensor conv_out_9_perm_0 = const()[name = tensor("conv_out_9_perm_0"), val = tensor([0, 2, 1])]; - tensor var_775_begin_0 = const()[name = tensor("op_775_begin_0"), val = tensor([0, -1, 0])]; - tensor var_775_end_0 = const()[name = tensor("op_775_end_0"), val = tensor([1, 16, 256])]; - tensor var_775_end_mask_0 = const()[name = tensor("op_775_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_9 = transpose(perm = conv_out_9_perm_0, x = input_99)[name = tensor("transpose_61")]; - tensor var_775 = slice_by_index(begin = var_775_begin_0, end = var_775_end_0, end_mask = var_775_end_mask_0, x = conv_out_9)[name = tensor("op_775")]; - tensor var_778_begin_0 = const()[name = tensor("op_778_begin_0"), val = tensor([0, 1, 0])]; - tensor var_778_end_0 = const()[name = tensor("op_778_end_0"), val = tensor([1, 2, 256])]; - tensor var_778_end_mask_0 = const()[name = tensor("op_778_end_mask_0"), val = tensor([true, false, true])]; - tensor var_778 = slice_by_index(begin = var_778_begin_0, end = var_778_end_0, end_mask = var_778_end_mask_0, x = x_15)[name = tensor("op_778")]; - tensor var_781_begin_0 = const()[name = tensor("op_781_begin_0"), val = tensor([0, 1, 0])]; - tensor var_781_end_0 = const()[name = tensor("op_781_end_0"), val = tensor([1, 16, 256])]; - tensor var_781_end_mask_0 = const()[name = tensor("op_781_end_mask_0"), val = tensor([true, true, true])]; - tensor var_781 = slice_by_index(begin = var_781_begin_0, end = var_781_end_0, end_mask = var_781_end_mask_0, x = input_91)[name = tensor("op_781")]; + tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_533)[name = tensor("x_expanded_9")]; + tensor var_547 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_547")]; + tensor var_548 = mul(x = window_11, y = var_547)[name = tensor("op_548")]; + tensor var_549 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_549")]; + tensor filled_window_9 = add(x = var_548, y = var_549)[name = tensor("filled_window_9")]; + tensor var_551 = greater_equal(x = filled_9, y = var_16)[name = tensor("op_551")]; + tensor var_552_dtype_0 = const()[name = tensor("op_552_dtype_0"), val = tensor("fp32")]; + tensor var_553 = const()[name = tensor("op_553"), val = tensor([1, 1, 1])]; + tensor var_552 = cast(dtype = var_552_dtype_0, x = var_551)[name = tensor("cast_166")]; + tensor use_roll_9 = reshape(shape = var_553, x = var_552)[name = tensor("use_roll_9")]; + tensor var_555 = sub(x = var_17, y = use_roll_9)[name = tensor("op_555")]; + tensor var_556 = mul(x = filled_window_9, y = var_555)[name = tensor("op_556")]; + tensor var_557 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_557")]; + tensor window_13 = add(x = var_556, y = var_557)[name = tensor("window_13")]; + tensor var_559 = const()[name = tensor("op_559"), val = tensor(0x1p+0)]; + tensor var_560 = add(x = filled_9, y = var_559)[name = tensor("op_560")]; + tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_560)[name = tensor("clip_9")]; + tensor var_563_begin_0 = const()[name = tensor("op_563_begin_0"), val = tensor([0, 1, 0])]; + tensor var_563_end_0 = const()[name = tensor("op_563_end_0"), val = tensor([1, 2, 256])]; + tensor var_563_end_mask_0 = const()[name = tensor("op_563_end_mask_0"), val = tensor([true, false, true])]; + tensor var_563 = slice_by_index(begin = var_563_begin_0, end = var_563_end_0, end_mask = var_563_end_mask_0, x = x_9)[name = tensor("op_563")]; + tensor var_566_begin_0 = const()[name = tensor("op_566_begin_0"), val = tensor([0, 1, 0])]; + tensor var_566_end_0 = const()[name = tensor("op_566_end_0"), val = tensor([1, 16, 256])]; + tensor var_566_end_mask_0 = const()[name = tensor("op_566_end_mask_0"), val = tensor([true, true, true])]; + tensor var_566 = slice_by_index(begin = var_566_begin_0, end = var_566_end_0, end_mask = var_566_end_mask_0, x = window_13)[name = tensor("op_566")]; tensor rolled_11_interleave_0 = const()[name = tensor("rolled_11_interleave_0"), val = tensor(false)]; - tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_781, var_778))[name = tensor("rolled_11")]; + tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_566, var_563))[name = tensor("rolled_11")]; tensor clip_10 = clip(alpha = var_21, beta = var_20, x = clip_9)[name = tensor("clip_10")]; - tensor var_786 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_786")]; - tensor var_787_dtype_0 = const()[name = tensor("op_787_dtype_0"), val = tensor("fp32")]; - tensor var_788 = const()[name = tensor("op_788"), val = tensor([1, 16, 1])]; - tensor var_787 = cast(dtype = var_787_dtype_0, x = var_786)[name = tensor("cast_165")]; - tensor fill_mask_11 = reshape(shape = var_788, x = var_787)[name = tensor("fill_mask_11")]; + tensor var_571 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_571")]; + tensor var_572_dtype_0 = const()[name = tensor("op_572_dtype_0"), val = tensor("fp32")]; + tensor var_573 = const()[name = tensor("op_573"), val = tensor([1, 16, 1])]; + tensor var_572 = cast(dtype = var_572_dtype_0, x = var_571)[name = tensor("cast_165")]; + tensor fill_mask_11 = reshape(shape = var_573, x = var_572)[name = tensor("fill_mask_11")]; tensor x_expanded_11_reps_0 = const()[name = tensor("x_expanded_11_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_778)[name = tensor("x_expanded_11")]; - tensor var_792 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_792")]; - tensor var_793 = mul(x = input_91, y = var_792)[name = tensor("op_793")]; - tensor var_794 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_794")]; - tensor filled_window_11 = add(x = var_793, y = var_794)[name = tensor("filled_window_11")]; - tensor var_796 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_796")]; - tensor var_797_dtype_0 = const()[name = tensor("op_797_dtype_0"), val = tensor("fp32")]; - tensor var_798 = const()[name = tensor("op_798"), val = tensor([1, 1, 1])]; - tensor var_797 = cast(dtype = var_797_dtype_0, x = var_796)[name = tensor("cast_164")]; - tensor use_roll_11 = reshape(shape = var_798, x = var_797)[name = tensor("use_roll_11")]; - tensor var_800 = sub(x = var_17, y = use_roll_11)[name = tensor("op_800")]; - tensor var_801 = mul(x = filled_window_11, y = var_800)[name = tensor("op_801")]; - tensor var_802 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_802")]; - tensor input_101 = add(x = var_801, y = var_802)[name = tensor("input_101")]; - tensor var_804 = const()[name = tensor("op_804"), val = tensor(0x1p+0)]; - tensor var_805 = add(x = clip_9, y = var_804)[name = tensor("op_805")]; - tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_805)[name = tensor("clip_11")]; - tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; - tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_101)[name = tensor("x_19")]; - tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_37_pad_type_0 = const()[name = tensor("inputs_37_pad_type_0"), val = tensor("valid")]; - tensor inputs_37_strides_0 = const()[name = tensor("inputs_37_strides_0"), val = tensor([1])]; - tensor inputs_37_pad_0 = const()[name = tensor("inputs_37_pad_0"), val = tensor([0, 0])]; - tensor inputs_37_dilations_0 = const()[name = tensor("inputs_37_dilations_0"), val = tensor([1])]; - tensor inputs_37_groups_0 = const()[name = tensor("inputs_37_groups_0"), val = tensor(1)]; - tensor input_103 = transpose(perm = input_103_perm_0, x = x_19)[name = tensor("transpose_60")]; - tensor inputs_37 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_37_dilations_0, groups = inputs_37_groups_0, pad = inputs_37_pad_0, pad_type = inputs_37_pad_type_0, strides = inputs_37_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_37")]; - tensor var_826_split_sizes_0 = const()[name = tensor("op_826_split_sizes_0"), val = tensor([256, 256])]; - tensor var_826_axis_0 = const()[name = tensor("op_826_axis_0"), val = tensor(1)]; - tensor var_826_0, tensor var_826_1 = split(axis = var_826_axis_0, split_sizes = var_826_split_sizes_0, x = inputs_37)[name = tensor("op_826")]; - tensor var_828 = sigmoid(x = var_826_1)[name = tensor("op_828")]; - tensor inputs_39 = mul(x = var_826_0, y = var_828)[name = tensor("inputs_39")]; - tensor outputs_aug_11_pad_type_0 = const()[name = tensor("outputs_aug_11_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_11_pad_0 = const()[name = tensor("outputs_aug_11_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_11_groups_0 = const()[name = tensor("outputs_aug_11_groups_0"), val = tensor(256)]; - tensor outputs_aug_11_strides_0 = const()[name = tensor("outputs_aug_11_strides_0"), val = tensor([1])]; - tensor outputs_aug_11_dilations_0 = const()[name = tensor("outputs_aug_11_dilations_0"), val = tensor([1])]; - tensor outputs_aug_11 = conv(dilations = outputs_aug_11_dilations_0, groups = outputs_aug_11_groups_0, pad = outputs_aug_11_pad_0, pad_type = outputs_aug_11_pad_type_0, strides = outputs_aug_11_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_39)[name = tensor("outputs_aug_11")]; - tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; - tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([1, 256, 16])]; - tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; - tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_11)[name = tensor("input_105")]; - tensor inputs_41 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_105)[name = tensor("inputs_41")]; - tensor input_107 = silu(x = inputs_41)[name = tensor("input_107")]; - tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; - tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; - tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; - tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; - tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; - tensor input_109 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; - tensor conv_out_11_perm_0 = const()[name = tensor("conv_out_11_perm_0"), val = tensor([0, 2, 1])]; - tensor var_859_begin_0 = const()[name = tensor("op_859_begin_0"), val = tensor([0, -1, 0])]; - tensor var_859_end_0 = const()[name = tensor("op_859_end_0"), val = tensor([1, 16, 256])]; - tensor var_859_end_mask_0 = const()[name = tensor("op_859_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_11 = transpose(perm = conv_out_11_perm_0, x = input_109)[name = tensor("transpose_59")]; - tensor var_859 = slice_by_index(begin = var_859_begin_0, end = var_859_end_0, end_mask = var_859_end_mask_0, x = conv_out_11)[name = tensor("op_859")]; - tensor var_862_begin_0 = const()[name = tensor("op_862_begin_0"), val = tensor([0, 2, 0])]; - tensor var_862_end_0 = const()[name = tensor("op_862_end_0"), val = tensor([1, 3, 256])]; - tensor var_862_end_mask_0 = const()[name = tensor("op_862_end_mask_0"), val = tensor([true, false, true])]; - tensor var_862 = slice_by_index(begin = var_862_begin_0, end = var_862_end_0, end_mask = var_862_end_mask_0, x = x_15)[name = tensor("op_862")]; - tensor var_865_begin_0 = const()[name = tensor("op_865_begin_0"), val = tensor([0, 1, 0])]; - tensor var_865_end_0 = const()[name = tensor("op_865_end_0"), val = tensor([1, 16, 256])]; - tensor var_865_end_mask_0 = const()[name = tensor("op_865_end_mask_0"), val = tensor([true, true, true])]; - tensor var_865 = slice_by_index(begin = var_865_begin_0, end = var_865_end_0, end_mask = var_865_end_mask_0, x = input_101)[name = tensor("op_865")]; + tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_563)[name = tensor("x_expanded_11")]; + tensor var_577 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_577")]; + tensor var_578 = mul(x = window_13, y = var_577)[name = tensor("op_578")]; + tensor var_579 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_579")]; + tensor filled_window_11 = add(x = var_578, y = var_579)[name = tensor("filled_window_11")]; + tensor var_581 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_581")]; + tensor var_582_dtype_0 = const()[name = tensor("op_582_dtype_0"), val = tensor("fp32")]; + tensor var_583 = const()[name = tensor("op_583"), val = tensor([1, 1, 1])]; + tensor var_582 = cast(dtype = var_582_dtype_0, x = var_581)[name = tensor("cast_164")]; + tensor use_roll_11 = reshape(shape = var_583, x = var_582)[name = tensor("use_roll_11")]; + tensor var_585 = sub(x = var_17, y = use_roll_11)[name = tensor("op_585")]; + tensor var_586 = mul(x = filled_window_11, y = var_585)[name = tensor("op_586")]; + tensor var_587 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_587")]; + tensor window_15 = add(x = var_586, y = var_587)[name = tensor("window_15")]; + tensor var_589 = const()[name = tensor("op_589"), val = tensor(0x1p+0)]; + tensor var_590 = add(x = clip_9, y = var_589)[name = tensor("op_590")]; + tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_590)[name = tensor("clip_11")]; + tensor var_593_begin_0 = const()[name = tensor("op_593_begin_0"), val = tensor([0, 2, 0])]; + tensor var_593_end_0 = const()[name = tensor("op_593_end_0"), val = tensor([1, 3, 256])]; + tensor var_593_end_mask_0 = const()[name = tensor("op_593_end_mask_0"), val = tensor([true, false, true])]; + tensor var_593 = slice_by_index(begin = var_593_begin_0, end = var_593_end_0, end_mask = var_593_end_mask_0, x = x_9)[name = tensor("op_593")]; + tensor var_596_begin_0 = const()[name = tensor("op_596_begin_0"), val = tensor([0, 1, 0])]; + tensor var_596_end_0 = const()[name = tensor("op_596_end_0"), val = tensor([1, 16, 256])]; + tensor var_596_end_mask_0 = const()[name = tensor("op_596_end_mask_0"), val = tensor([true, true, true])]; + tensor var_596 = slice_by_index(begin = var_596_begin_0, end = var_596_end_0, end_mask = var_596_end_mask_0, x = window_15)[name = tensor("op_596")]; tensor rolled_13_interleave_0 = const()[name = tensor("rolled_13_interleave_0"), val = tensor(false)]; - tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_865, var_862))[name = tensor("rolled_13")]; + tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_596, var_593))[name = tensor("rolled_13")]; tensor clip_12 = clip(alpha = var_21, beta = var_20, x = clip_11)[name = tensor("clip_12")]; - tensor var_870 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_870")]; - tensor var_871_dtype_0 = const()[name = tensor("op_871_dtype_0"), val = tensor("fp32")]; - tensor var_872 = const()[name = tensor("op_872"), val = tensor([1, 16, 1])]; - tensor var_871 = cast(dtype = var_871_dtype_0, x = var_870)[name = tensor("cast_163")]; - tensor fill_mask_13 = reshape(shape = var_872, x = var_871)[name = tensor("fill_mask_13")]; + tensor var_601 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_601")]; + tensor var_602_dtype_0 = const()[name = tensor("op_602_dtype_0"), val = tensor("fp32")]; + tensor var_603 = const()[name = tensor("op_603"), val = tensor([1, 16, 1])]; + tensor var_602 = cast(dtype = var_602_dtype_0, x = var_601)[name = tensor("cast_163")]; + tensor fill_mask_13 = reshape(shape = var_603, x = var_602)[name = tensor("fill_mask_13")]; tensor x_expanded_13_reps_0 = const()[name = tensor("x_expanded_13_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_862)[name = tensor("x_expanded_13")]; - tensor var_876 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_876")]; - tensor var_877 = mul(x = input_101, y = var_876)[name = tensor("op_877")]; - tensor var_878 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_878")]; - tensor filled_window_13 = add(x = var_877, y = var_878)[name = tensor("filled_window_13")]; - tensor var_880 = greater_equal(x = clip_11, y = var_16)[name = tensor("op_880")]; - tensor var_881_dtype_0 = const()[name = tensor("op_881_dtype_0"), val = tensor("fp32")]; - tensor var_882 = const()[name = tensor("op_882"), val = tensor([1, 1, 1])]; - tensor var_881 = cast(dtype = var_881_dtype_0, x = var_880)[name = tensor("cast_162")]; - tensor use_roll_13 = reshape(shape = var_882, x = var_881)[name = tensor("use_roll_13")]; - tensor var_884 = sub(x = var_17, y = use_roll_13)[name = tensor("op_884")]; - tensor var_885 = mul(x = filled_window_13, y = var_884)[name = tensor("op_885")]; - tensor var_886 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_886")]; - tensor input_111 = add(x = var_885, y = var_886)[name = tensor("input_111")]; - tensor var_888 = const()[name = tensor("op_888"), val = tensor(0x1p+0)]; - tensor var_889 = add(x = clip_11, y = var_888)[name = tensor("op_889")]; - tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_889)[name = tensor("clip_13")]; - tensor x_21_axes_0 = const()[name = tensor("x_21_axes_0"), val = tensor([-1])]; - tensor x_21 = layer_norm(axes = x_21_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_111)[name = tensor("x_21")]; - tensor input_113_perm_0 = const()[name = tensor("input_113_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_43_pad_type_0 = const()[name = tensor("inputs_43_pad_type_0"), val = tensor("valid")]; - tensor inputs_43_strides_0 = const()[name = tensor("inputs_43_strides_0"), val = tensor([1])]; - tensor inputs_43_pad_0 = const()[name = tensor("inputs_43_pad_0"), val = tensor([0, 0])]; - tensor inputs_43_dilations_0 = const()[name = tensor("inputs_43_dilations_0"), val = tensor([1])]; - tensor inputs_43_groups_0 = const()[name = tensor("inputs_43_groups_0"), val = tensor(1)]; - tensor input_113 = transpose(perm = input_113_perm_0, x = x_21)[name = tensor("transpose_58")]; - tensor inputs_43 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_43_dilations_0, groups = inputs_43_groups_0, pad = inputs_43_pad_0, pad_type = inputs_43_pad_type_0, strides = inputs_43_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_113)[name = tensor("inputs_43")]; - tensor var_910_split_sizes_0 = const()[name = tensor("op_910_split_sizes_0"), val = tensor([256, 256])]; - tensor var_910_axis_0 = const()[name = tensor("op_910_axis_0"), val = tensor(1)]; - tensor var_910_0, tensor var_910_1 = split(axis = var_910_axis_0, split_sizes = var_910_split_sizes_0, x = inputs_43)[name = tensor("op_910")]; - tensor var_912 = sigmoid(x = var_910_1)[name = tensor("op_912")]; - tensor inputs_45 = mul(x = var_910_0, y = var_912)[name = tensor("inputs_45")]; - tensor outputs_aug_13_pad_type_0 = const()[name = tensor("outputs_aug_13_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_13_pad_0 = const()[name = tensor("outputs_aug_13_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_13_groups_0 = const()[name = tensor("outputs_aug_13_groups_0"), val = tensor(256)]; - tensor outputs_aug_13_strides_0 = const()[name = tensor("outputs_aug_13_strides_0"), val = tensor([1])]; - tensor outputs_aug_13_dilations_0 = const()[name = tensor("outputs_aug_13_dilations_0"), val = tensor([1])]; - tensor outputs_aug_13 = conv(dilations = outputs_aug_13_dilations_0, groups = outputs_aug_13_groups_0, pad = outputs_aug_13_pad_0, pad_type = outputs_aug_13_pad_type_0, strides = outputs_aug_13_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_45)[name = tensor("outputs_aug_13")]; - tensor input_115_begin_0 = const()[name = tensor("input_115_begin_0"), val = tensor([0, 0, 0])]; - tensor input_115_end_0 = const()[name = tensor("input_115_end_0"), val = tensor([1, 256, 16])]; - tensor input_115_end_mask_0 = const()[name = tensor("input_115_end_mask_0"), val = tensor([true, true, false])]; - tensor input_115 = slice_by_index(begin = input_115_begin_0, end = input_115_end_0, end_mask = input_115_end_mask_0, x = outputs_aug_13)[name = tensor("input_115")]; - tensor inputs_47 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_115)[name = tensor("inputs_47")]; - tensor input_117 = silu(x = inputs_47)[name = tensor("input_117")]; - tensor input_119_pad_type_0 = const()[name = tensor("input_119_pad_type_0"), val = tensor("valid")]; - tensor input_119_strides_0 = const()[name = tensor("input_119_strides_0"), val = tensor([1])]; - tensor input_119_pad_0 = const()[name = tensor("input_119_pad_0"), val = tensor([0, 0])]; - tensor input_119_dilations_0 = const()[name = tensor("input_119_dilations_0"), val = tensor([1])]; - tensor input_119_groups_0 = const()[name = tensor("input_119_groups_0"), val = tensor(1)]; - tensor input_119 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_119_dilations_0, groups = input_119_groups_0, pad = input_119_pad_0, pad_type = input_119_pad_type_0, strides = input_119_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_117)[name = tensor("input_119")]; - tensor conv_out_13_perm_0 = const()[name = tensor("conv_out_13_perm_0"), val = tensor([0, 2, 1])]; - tensor var_943_begin_0 = const()[name = tensor("op_943_begin_0"), val = tensor([0, -1, 0])]; - tensor var_943_end_0 = const()[name = tensor("op_943_end_0"), val = tensor([1, 16, 256])]; - tensor var_943_end_mask_0 = const()[name = tensor("op_943_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_13 = transpose(perm = conv_out_13_perm_0, x = input_119)[name = tensor("transpose_57")]; - tensor var_943 = slice_by_index(begin = var_943_begin_0, end = var_943_end_0, end_mask = var_943_end_mask_0, x = conv_out_13)[name = tensor("op_943")]; - tensor var_946_begin_0 = const()[name = tensor("op_946_begin_0"), val = tensor([0, 3, 0])]; - tensor var_946_end_0 = const()[name = tensor("op_946_end_0"), val = tensor([1, 1, 256])]; - tensor var_946_end_mask_0 = const()[name = tensor("op_946_end_mask_0"), val = tensor([true, true, true])]; - tensor var_946 = slice_by_index(begin = var_946_begin_0, end = var_946_end_0, end_mask = var_946_end_mask_0, x = x_15)[name = tensor("op_946")]; - tensor var_949_begin_0 = const()[name = tensor("op_949_begin_0"), val = tensor([0, 1, 0])]; - tensor var_949_end_0 = const()[name = tensor("op_949_end_0"), val = tensor([1, 16, 256])]; - tensor var_949_end_mask_0 = const()[name = tensor("op_949_end_mask_0"), val = tensor([true, true, true])]; - tensor var_949 = slice_by_index(begin = var_949_begin_0, end = var_949_end_0, end_mask = var_949_end_mask_0, x = input_111)[name = tensor("op_949")]; + tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_593)[name = tensor("x_expanded_13")]; + tensor var_607 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_607")]; + tensor var_608 = mul(x = window_15, y = var_607)[name = tensor("op_608")]; + tensor var_609 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_609")]; + tensor filled_window_13 = add(x = var_608, y = var_609)[name = tensor("filled_window_13")]; + tensor var_611 = greater_equal(x = clip_11, y = var_16)[name = tensor("op_611")]; + tensor var_612_dtype_0 = const()[name = tensor("op_612_dtype_0"), val = tensor("fp32")]; + tensor var_613 = const()[name = tensor("op_613"), val = tensor([1, 1, 1])]; + tensor var_612 = cast(dtype = var_612_dtype_0, x = var_611)[name = tensor("cast_162")]; + tensor use_roll_13 = reshape(shape = var_613, x = var_612)[name = tensor("use_roll_13")]; + tensor var_615 = sub(x = var_17, y = use_roll_13)[name = tensor("op_615")]; + tensor var_616 = mul(x = filled_window_13, y = var_615)[name = tensor("op_616")]; + tensor var_617 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_617")]; + tensor window_17 = add(x = var_616, y = var_617)[name = tensor("window_17")]; + tensor var_619 = const()[name = tensor("op_619"), val = tensor(0x1p+0)]; + tensor var_620 = add(x = clip_11, y = var_619)[name = tensor("op_620")]; + tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_620)[name = tensor("clip_13")]; + tensor var_623_begin_0 = const()[name = tensor("op_623_begin_0"), val = tensor([0, 3, 0])]; + tensor var_623_end_0 = const()[name = tensor("op_623_end_0"), val = tensor([1, 1, 256])]; + tensor var_623_end_mask_0 = const()[name = tensor("op_623_end_mask_0"), val = tensor([true, true, true])]; + tensor var_623 = slice_by_index(begin = var_623_begin_0, end = var_623_end_0, end_mask = var_623_end_mask_0, x = x_9)[name = tensor("op_623")]; + tensor var_626_begin_0 = const()[name = tensor("op_626_begin_0"), val = tensor([0, 1, 0])]; + tensor var_626_end_0 = const()[name = tensor("op_626_end_0"), val = tensor([1, 16, 256])]; + tensor var_626_end_mask_0 = const()[name = tensor("op_626_end_mask_0"), val = tensor([true, true, true])]; + tensor var_626 = slice_by_index(begin = var_626_begin_0, end = var_626_end_0, end_mask = var_626_end_mask_0, x = window_17)[name = tensor("op_626")]; tensor rolled_15_interleave_0 = const()[name = tensor("rolled_15_interleave_0"), val = tensor(false)]; - tensor rolled_15 = concat(axis = var_33, interleave = rolled_15_interleave_0, values = (var_949, var_946))[name = tensor("rolled_15")]; + tensor rolled_15 = concat(axis = var_33, interleave = rolled_15_interleave_0, values = (var_626, var_623))[name = tensor("rolled_15")]; tensor clip_14 = clip(alpha = var_21, beta = var_20, x = clip_13)[name = tensor("clip_14")]; - tensor var_954 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_954")]; - tensor var_955_dtype_0 = const()[name = tensor("op_955_dtype_0"), val = tensor("fp32")]; - tensor var_956 = const()[name = tensor("op_956"), val = tensor([1, 16, 1])]; - tensor var_955 = cast(dtype = var_955_dtype_0, x = var_954)[name = tensor("cast_161")]; - tensor fill_mask_15 = reshape(shape = var_956, x = var_955)[name = tensor("fill_mask_15")]; + tensor var_631 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_631")]; + tensor var_632_dtype_0 = const()[name = tensor("op_632_dtype_0"), val = tensor("fp32")]; + tensor var_633 = const()[name = tensor("op_633"), val = tensor([1, 16, 1])]; + tensor var_632 = cast(dtype = var_632_dtype_0, x = var_631)[name = tensor("cast_161")]; + tensor fill_mask_15 = reshape(shape = var_633, x = var_632)[name = tensor("fill_mask_15")]; tensor x_expanded_15_reps_0 = const()[name = tensor("x_expanded_15_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_946)[name = tensor("x_expanded_15")]; - tensor var_960 = sub(x = var_17, y = fill_mask_15)[name = tensor("op_960")]; - tensor var_961 = mul(x = input_111, y = var_960)[name = tensor("op_961")]; - tensor var_962 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_962")]; - tensor filled_window_15 = add(x = var_961, y = var_962)[name = tensor("filled_window_15")]; - tensor var_964 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_964")]; - tensor var_965_dtype_0 = const()[name = tensor("op_965_dtype_0"), val = tensor("fp32")]; - tensor var_966 = const()[name = tensor("op_966"), val = tensor([1, 1, 1])]; - tensor var_965 = cast(dtype = var_965_dtype_0, x = var_964)[name = tensor("cast_160")]; - tensor use_roll_15 = reshape(shape = var_966, x = var_965)[name = tensor("use_roll_15")]; - tensor var_968 = sub(x = var_17, y = use_roll_15)[name = tensor("op_968")]; - tensor var_969 = mul(x = filled_window_15, y = var_968)[name = tensor("op_969")]; - tensor var_970 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_970")]; - tensor input_121 = add(x = var_969, y = var_970)[name = tensor("input_121")]; - tensor var_972 = const()[name = tensor("op_972"), val = tensor(0x1p+0)]; - tensor var_973 = add(x = clip_13, y = var_972)[name = tensor("op_973")]; - tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_973)[name = tensor("clip_15")]; - tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; - tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_121)[name = tensor("x_23")]; - tensor input_123_perm_0 = const()[name = tensor("input_123_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_49_pad_type_0 = const()[name = tensor("inputs_49_pad_type_0"), val = tensor("valid")]; - tensor inputs_49_strides_0 = const()[name = tensor("inputs_49_strides_0"), val = tensor([1])]; - tensor inputs_49_pad_0 = const()[name = tensor("inputs_49_pad_0"), val = tensor([0, 0])]; - tensor inputs_49_dilations_0 = const()[name = tensor("inputs_49_dilations_0"), val = tensor([1])]; - tensor inputs_49_groups_0 = const()[name = tensor("inputs_49_groups_0"), val = tensor(1)]; - tensor input_123 = transpose(perm = input_123_perm_0, x = x_23)[name = tensor("transpose_56")]; - tensor inputs_49 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_49_dilations_0, groups = inputs_49_groups_0, pad = inputs_49_pad_0, pad_type = inputs_49_pad_type_0, strides = inputs_49_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_123)[name = tensor("inputs_49")]; - tensor var_994_split_sizes_0 = const()[name = tensor("op_994_split_sizes_0"), val = tensor([256, 256])]; - tensor var_994_axis_0 = const()[name = tensor("op_994_axis_0"), val = tensor(1)]; - tensor var_994_0, tensor var_994_1 = split(axis = var_994_axis_0, split_sizes = var_994_split_sizes_0, x = inputs_49)[name = tensor("op_994")]; - tensor var_996 = sigmoid(x = var_994_1)[name = tensor("op_996")]; - tensor inputs_51 = mul(x = var_994_0, y = var_996)[name = tensor("inputs_51")]; - tensor outputs_aug_15_pad_type_0 = const()[name = tensor("outputs_aug_15_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_15_pad_0 = const()[name = tensor("outputs_aug_15_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_15_groups_0 = const()[name = tensor("outputs_aug_15_groups_0"), val = tensor(256)]; - tensor outputs_aug_15_strides_0 = const()[name = tensor("outputs_aug_15_strides_0"), val = tensor([1])]; - tensor outputs_aug_15_dilations_0 = const()[name = tensor("outputs_aug_15_dilations_0"), val = tensor([1])]; - tensor outputs_aug_15 = conv(dilations = outputs_aug_15_dilations_0, groups = outputs_aug_15_groups_0, pad = outputs_aug_15_pad_0, pad_type = outputs_aug_15_pad_type_0, strides = outputs_aug_15_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_51)[name = tensor("outputs_aug_15")]; - tensor input_125_begin_0 = const()[name = tensor("input_125_begin_0"), val = tensor([0, 0, 0])]; - tensor input_125_end_0 = const()[name = tensor("input_125_end_0"), val = tensor([1, 256, 16])]; - tensor input_125_end_mask_0 = const()[name = tensor("input_125_end_mask_0"), val = tensor([true, true, false])]; - tensor input_125 = slice_by_index(begin = input_125_begin_0, end = input_125_end_0, end_mask = input_125_end_mask_0, x = outputs_aug_15)[name = tensor("input_125")]; - tensor inputs_53 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_125)[name = tensor("inputs_53")]; - tensor input_127 = silu(x = inputs_53)[name = tensor("input_127")]; - tensor input_129_pad_type_0 = const()[name = tensor("input_129_pad_type_0"), val = tensor("valid")]; - tensor input_129_strides_0 = const()[name = tensor("input_129_strides_0"), val = tensor([1])]; - tensor input_129_pad_0 = const()[name = tensor("input_129_pad_0"), val = tensor([0, 0])]; - tensor input_129_dilations_0 = const()[name = tensor("input_129_dilations_0"), val = tensor([1])]; - tensor input_129_groups_0 = const()[name = tensor("input_129_groups_0"), val = tensor(1)]; - tensor input_129 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_129_dilations_0, groups = input_129_groups_0, pad = input_129_pad_0, pad_type = input_129_pad_type_0, strides = input_129_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_127)[name = tensor("input_129")]; - tensor conv_out_15_perm_0 = const()[name = tensor("conv_out_15_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1027_begin_0 = const()[name = tensor("op_1027_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1027_end_0 = const()[name = tensor("op_1027_end_0"), val = tensor([1, 16, 256])]; - tensor var_1027_end_mask_0 = const()[name = tensor("op_1027_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_15 = transpose(perm = conv_out_15_perm_0, x = input_129)[name = tensor("transpose_55")]; - tensor var_1027 = slice_by_index(begin = var_1027_begin_0, end = var_1027_end_0, end_mask = var_1027_end_mask_0, x = conv_out_15)[name = tensor("op_1027")]; - tensor var_1030_interleave_0 = const()[name = tensor("op_1030_interleave_0"), val = tensor(false)]; - tensor var_1030 = concat(axis = var_33, interleave = var_1030_interleave_0, values = (var_775, var_859, var_943, var_1027))[name = tensor("op_1030")]; - tensor input_131 = add(x = x_15, y = var_1030)[name = tensor("input_131")]; - tensor input_133_axes_0 = const()[name = tensor("input_133_axes_0"), val = tensor([-1])]; - tensor input_133 = layer_norm(axes = input_133_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_131)[name = tensor("input_133")]; - tensor inputs_55 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_133)[name = tensor("linear_17")]; - tensor input_135 = silu(x = inputs_55)[name = tensor("input_135")]; - tensor input_139 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_135)[name = tensor("linear_18")]; - tensor var_1053 = const()[name = tensor("op_1053"), val = tensor(0x1p-1)]; - tensor var_1054 = mul(x = input_139, y = var_1053)[name = tensor("op_1054")]; - tensor input_141 = add(x = var_1054, y = input_131)[name = tensor("input_141")]; - tensor input_143_axes_0 = const()[name = tensor("input_143_axes_0"), val = tensor([-1])]; - tensor input_143 = layer_norm(axes = input_143_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_141)[name = tensor("input_143")]; - tensor input_145_axes_0 = const()[name = tensor("input_145_axes_0"), val = tensor([-1])]; - tensor input_145 = layer_norm(axes = input_145_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_143)[name = tensor("input_145")]; - tensor inputs_57 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_145)[name = tensor("linear_19")]; - tensor input_147 = silu(x = inputs_57)[name = tensor("input_147")]; - tensor input_151 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_147)[name = tensor("linear_20")]; - tensor var_1083 = const()[name = tensor("op_1083"), val = tensor(0x1p-1)]; - tensor var_1084 = mul(x = input_151, y = var_1083)[name = tensor("op_1084")]; - tensor input_153 = add(x = var_1084, y = input_143)[name = tensor("input_153")]; - tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; - tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_153)[name = tensor("x_25")]; + tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_623)[name = tensor("x_expanded_15")]; + tensor var_637 = sub(x = var_17, y = fill_mask_15)[name = tensor("op_637")]; + tensor var_638 = mul(x = window_17, y = var_637)[name = tensor("op_638")]; + tensor var_639 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_639")]; + tensor filled_window_15 = add(x = var_638, y = var_639)[name = tensor("filled_window_15")]; + tensor var_641 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_641")]; + tensor var_642_dtype_0 = const()[name = tensor("op_642_dtype_0"), val = tensor("fp32")]; + tensor var_643 = const()[name = tensor("op_643"), val = tensor([1, 1, 1])]; + tensor var_642 = cast(dtype = var_642_dtype_0, x = var_641)[name = tensor("cast_160")]; + tensor use_roll_15 = reshape(shape = var_643, x = var_642)[name = tensor("use_roll_15")]; + tensor var_645 = sub(x = var_17, y = use_roll_15)[name = tensor("op_645")]; + tensor var_646 = mul(x = filled_window_15, y = var_645)[name = tensor("op_646")]; + tensor var_647 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_647")]; + tensor window_19 = add(x = var_646, y = var_647)[name = tensor("window_19")]; + tensor var_649 = const()[name = tensor("op_649"), val = tensor(0x1p+0)]; + tensor var_650 = add(x = clip_13, y = var_649)[name = tensor("op_650")]; + tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_650)[name = tensor("clip_15")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_30, interleave = input_61_interleave_0, values = (window_13, window_15, window_17, window_19))[name = tensor("input_61")]; + tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; + tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; + tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_13_pad_type_0 = const()[name = tensor("inputs_13_pad_type_0"), val = tensor("valid")]; + tensor inputs_13_strides_0 = const()[name = tensor("inputs_13_strides_0"), val = tensor([1])]; + tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; + tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; + tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; + tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; + tensor var_673_split_sizes_0 = const()[name = tensor("op_673_split_sizes_0"), val = tensor([256, 256])]; + tensor var_673_axis_0 = const()[name = tensor("op_673_axis_0"), val = tensor(1)]; + tensor var_673_0, tensor var_673_1 = split(axis = var_673_axis_0, split_sizes = var_673_split_sizes_0, x = inputs_13)[name = tensor("op_673")]; + tensor var_675 = sigmoid(x = var_673_1)[name = tensor("op_675")]; + tensor inputs_15 = mul(x = var_673_0, y = var_675)[name = tensor("inputs_15")]; + tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; + tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; + tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; + tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_15)[name = tensor("outputs_aug_3")]; + tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; + tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([4, 256, 16])]; + tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; + tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_3)[name = tensor("input_65")]; + tensor inputs_17 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_65)[name = tensor("inputs_17")]; + tensor input_67 = silu(x = inputs_17)[name = tensor("input_67")]; + tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; + tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; + tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; + tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; + tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; + tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; + tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; + tensor var_706_begin_0 = const()[name = tensor("op_706_begin_0"), val = tensor([0, -1, 0])]; + tensor var_706_end_0 = const()[name = tensor("op_706_end_0"), val = tensor([4, 16, 256])]; + tensor var_706_end_mask_0 = const()[name = tensor("op_706_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_706 = slice_by_index(begin = var_706_begin_0, end = var_706_end_0, end_mask = var_706_end_mask_0, x = conv_out_3)[name = tensor("op_706")]; + tensor var_708_perm_0 = const()[name = tensor("op_708_perm_0"), val = tensor([1, 0, 2])]; + tensor var_708 = transpose(perm = var_708_perm_0, x = var_706)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_708)[name = tensor("input_71")]; + tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; + tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; + tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; + tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; + tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; + tensor var_731 = const()[name = tensor("op_731"), val = tensor(0x1p-1)]; + tensor var_732 = mul(x = input_79, y = var_731)[name = tensor("op_732")]; + tensor input_81 = add(x = var_732, y = input_71)[name = tensor("input_81")]; + tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; + tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; + tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; + tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; + tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; + tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; + tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; + tensor var_761 = const()[name = tensor("op_761"), val = tensor(0x1p-1)]; + tensor var_762 = mul(x = input_91, y = var_761)[name = tensor("op_762")]; + tensor input_93 = add(x = var_762, y = input_83)[name = tensor("input_93")]; + tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; + tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; tensor prev_kv_5_end_0 = const()[name = tensor("prev_kv_5_end_0"), val = tensor([3, 1, 4, 64, 64])]; tensor prev_kv_5_end_mask_0 = const()[name = tensor("prev_kv_5_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1029,389 +800,274 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_1098 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_25)[name = tensor("linear_21")]; - tensor var_1099 = const()[name = tensor("op_1099"), val = tensor([1, 4, 4, 64])]; - tensor var_1100 = reshape(shape = var_1099, x = var_1098)[name = tensor("op_1100")]; + tensor var_776 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_777 = const()[name = tensor("op_777"), val = tensor([1, 4, 4, 64])]; + tensor var_778 = reshape(shape = var_777, x = var_776)[name = tensor("op_778")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1104 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_25)[name = tensor("linear_22")]; - tensor var_1105 = const()[name = tensor("op_1105"), val = tensor(0x1p-3)]; - tensor var_1106 = mul(x = var_1104, y = var_1105)[name = tensor("op_1106")]; - tensor var_1107 = const()[name = tensor("op_1107"), val = tensor([1, 4, 4, 64])]; - tensor var_1108 = reshape(shape = var_1107, x = var_1106)[name = tensor("op_1108")]; + tensor var_782 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_783 = const()[name = tensor("op_783"), val = tensor(0x1p-3)]; + tensor var_784 = mul(x = var_782, y = var_783)[name = tensor("op_784")]; + tensor var_785 = const()[name = tensor("op_785"), val = tensor([1, 4, 4, 64])]; + tensor var_786 = reshape(shape = var_785, x = var_784)[name = tensor("op_786")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1112 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_25)[name = tensor("linear_23")]; - tensor var_1113 = const()[name = tensor("op_1113"), val = tensor([1, 4, 4, 64])]; - tensor var_1114 = reshape(shape = var_1113, x = var_1112)[name = tensor("op_1114")]; + tensor var_790 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_791 = const()[name = tensor("op_791"), val = tensor([1, 4, 4, 64])]; + tensor var_792 = reshape(shape = var_791, x = var_790)[name = tensor("op_792")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_157 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_25)[name = tensor("linear_24")]; + tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; tensor s_t_5 = add(x = prev_scale_5, y = encoder__t_index)[name = tensor("s_t_5")]; tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_1108)[name = tensor("transpose_53")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_1100)[name = tensor("transpose_54")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_786)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_778)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_1124 = const()[name = tensor("op_1124"), val = tensor([4, 1])]; - tensor var_1125 = reshape(shape = var_1124, x = sqrt_s_t_5)[name = tensor("op_1125")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_1125)[name = tensor("M_5")]; - tensor var_1127 = mul(x = qk_5, y = M_5)[name = tensor("op_1127")]; + tensor var_802 = const()[name = tensor("op_802"), val = tensor([4, 1])]; + tensor var_803 = reshape(shape = var_802, x = sqrt_s_t_5)[name = tensor("op_803")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_803)[name = tensor("M_5")]; + tensor var_805 = mul(x = qk_5, y = M_5)[name = tensor("op_805")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_1114)[name = tensor("transpose_52")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_1127, y = v_5)[name = tensor("inner_5")]; - tensor var_1129_transpose_x_0 = const()[name = tensor("op_1129_transpose_x_0"), val = tensor(false)]; - tensor var_1129_transpose_y_0 = const()[name = tensor("op_1129_transpose_y_0"), val = tensor(false)]; - tensor var_1129 = matmul(transpose_x = var_1129_transpose_x_0, transpose_y = var_1129_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_1129")]; - tensor var_1130 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_1130")]; - tensor var_1131 = const()[name = tensor("op_1131"), val = tensor([1, 1, 4, 1])]; - tensor var_1132 = reshape(shape = var_1131, x = var_1130)[name = tensor("op_1132")]; - tensor cross_5 = mul(x = var_1129, y = var_1132)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_792)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_805, y = v_5)[name = tensor("inner_5")]; + tensor var_807_transpose_x_0 = const()[name = tensor("op_807_transpose_x_0"), val = tensor(false)]; + tensor var_807_transpose_y_0 = const()[name = tensor("op_807_transpose_y_0"), val = tensor(false)]; + tensor var_807 = matmul(transpose_x = var_807_transpose_x_0, transpose_y = var_807_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_807")]; + tensor var_808 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_808")]; + tensor var_809 = const()[name = tensor("op_809"), val = tensor([1, 1, 4, 1])]; + tensor var_810 = reshape(shape = var_809, x = var_808)[name = tensor("op_810")]; + tensor cross_5 = mul(x = var_807, y = var_810)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_1135 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_1135")]; - tensor var_1137_transpose_x_1 = const()[name = tensor("op_1137_transpose_x_1"), val = tensor(true)]; - tensor var_1137_transpose_y_1 = const()[name = tensor("op_1137_transpose_y_1"), val = tensor(false)]; - tensor var_1137 = matmul(transpose_x = var_1137_transpose_x_1, transpose_y = var_1137_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_1137")]; - tensor new_kv_unnorm_5 = add(x = var_1135, y = var_1137)[name = tensor("new_kv_unnorm_5")]; - tensor var_1139 = const()[name = tensor("op_1139"), val = tensor(0x1p+2)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_1139)[name = tensor("new_scale_5")]; - tensor var_1141 = sqrt(x = new_scale_5)[name = tensor("op_1141")]; - tensor var_1142 = real_div(x = new_kv_unnorm_5, y = var_1141)[name = tensor("op_1142")]; - tensor var_1143_perm_0 = const()[name = tensor("op_1143_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_813 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_813")]; + tensor var_815_transpose_x_1 = const()[name = tensor("op_815_transpose_x_1"), val = tensor(true)]; + tensor var_815_transpose_y_1 = const()[name = tensor("op_815_transpose_y_1"), val = tensor(false)]; + tensor var_815 = matmul(transpose_x = var_815_transpose_x_1, transpose_y = var_815_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_815")]; + tensor new_kv_unnorm_5 = add(x = var_813, y = var_815)[name = tensor("new_kv_unnorm_5")]; + tensor var_817 = const()[name = tensor("op_817"), val = tensor(0x1p+2)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_817)[name = tensor("new_scale_5")]; + tensor var_819 = sqrt(x = new_scale_5)[name = tensor("op_819")]; + tensor var_820 = real_div(x = new_kv_unnorm_5, y = var_819)[name = tensor("op_820")]; + tensor var_821_perm_0 = const()[name = tensor("op_821_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_1143 = transpose(perm = var_1143_perm_0, x = out_13)[name = tensor("transpose_51")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_1143)[name = tensor("out_15")]; - tensor var_1147 = const()[name = tensor("op_1147"), val = tensor([1, 4, 256])]; - tensor out_17 = reshape(shape = var_1147, x = out_15)[name = tensor("out_17")]; - tensor var_1149 = silu(x = input_157)[name = tensor("op_1149")]; - tensor input_159 = mul(x = var_1149, y = out_17)[name = tensor("input_159")]; - tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_159)[name = tensor("linear_25")]; - tensor x_27 = add(x = input_153, y = ret_out_5)[name = tensor("x_27")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor var_821 = transpose(perm = var_821_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_821)[name = tensor("out_15")]; + tensor var_825 = const()[name = tensor("op_825"), val = tensor([1, 4, 256])]; + tensor out_17 = reshape(shape = var_825, x = out_15)[name = tensor("out_17")]; + tensor var_827 = silu(x = input_97)[name = tensor("op_827")]; + tensor input_99 = mul(x = var_827, y = out_17)[name = tensor("input_99")]; + tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; + tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; + tensor window_21_begin_0 = const()[name = tensor("window_21_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_21_end_0 = const()[name = tensor("window_21_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_21_end_mask_0 = const()[name = tensor("window_21_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_21_squeeze_mask_0 = const()[name = tensor("window_21_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_21 = slice_by_index(begin = window_21_begin_0, end = window_21_end_0, end_mask = window_21_end_mask_0, squeeze_mask = window_21_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_21")]; tensor filled_17_begin_0 = const()[name = tensor("filled_17_begin_0"), val = tensor([2, 0])]; tensor filled_17_end_0 = const()[name = tensor("filled_17_end_0"), val = tensor([3, 1])]; tensor filled_17_end_mask_0 = const()[name = tensor("filled_17_end_mask_0"), val = tensor([false, true])]; tensor filled_17_squeeze_mask_0 = const()[name = tensor("filled_17_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_17 = slice_by_index(begin = filled_17_begin_0, end = filled_17_end_0, end_mask = filled_17_end_mask_0, squeeze_mask = filled_17_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_17")]; - tensor var_1158_begin_0 = const()[name = tensor("op_1158_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1158_end_0 = const()[name = tensor("op_1158_end_0"), val = tensor([1, 1, 256])]; - tensor var_1158_end_mask_0 = const()[name = tensor("op_1158_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1158 = slice_by_index(begin = var_1158_begin_0, end = var_1158_end_0, end_mask = var_1158_end_mask_0, x = x_27)[name = tensor("op_1158")]; - tensor var_1161_begin_0 = const()[name = tensor("op_1161_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1161_end_0 = const()[name = tensor("op_1161_end_0"), val = tensor([1, 16, 256])]; - tensor var_1161_end_mask_0 = const()[name = tensor("op_1161_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1161 = slice_by_index(begin = var_1161_begin_0, end = var_1161_end_0, end_mask = var_1161_end_mask_0, x = window_5)[name = tensor("op_1161")]; + tensor var_836_begin_0 = const()[name = tensor("op_836_begin_0"), val = tensor([0, 0, 0])]; + tensor var_836_end_0 = const()[name = tensor("op_836_end_0"), val = tensor([1, 1, 256])]; + tensor var_836_end_mask_0 = const()[name = tensor("op_836_end_mask_0"), val = tensor([true, false, true])]; + tensor var_836 = slice_by_index(begin = var_836_begin_0, end = var_836_end_0, end_mask = var_836_end_mask_0, x = x_15)[name = tensor("op_836")]; + tensor var_839_begin_0 = const()[name = tensor("op_839_begin_0"), val = tensor([0, 1, 0])]; + tensor var_839_end_0 = const()[name = tensor("op_839_end_0"), val = tensor([1, 16, 256])]; + tensor var_839_end_mask_0 = const()[name = tensor("op_839_end_mask_0"), val = tensor([true, true, true])]; + tensor var_839 = slice_by_index(begin = var_839_begin_0, end = var_839_end_0, end_mask = var_839_end_mask_0, x = window_21)[name = tensor("op_839")]; tensor rolled_17_interleave_0 = const()[name = tensor("rolled_17_interleave_0"), val = tensor(false)]; - tensor rolled_17 = concat(axis = var_33, interleave = rolled_17_interleave_0, values = (var_1161, var_1158))[name = tensor("rolled_17")]; + tensor rolled_17 = concat(axis = var_33, interleave = rolled_17_interleave_0, values = (var_839, var_836))[name = tensor("rolled_17")]; tensor clip_16 = clip(alpha = var_21, beta = var_20, x = filled_17)[name = tensor("clip_16")]; - tensor var_1166 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_1166")]; - tensor var_1167_dtype_0 = const()[name = tensor("op_1167_dtype_0"), val = tensor("fp32")]; - tensor var_1168 = const()[name = tensor("op_1168"), val = tensor([1, 16, 1])]; - tensor var_1167 = cast(dtype = var_1167_dtype_0, x = var_1166)[name = tensor("cast_159")]; - tensor fill_mask_17 = reshape(shape = var_1168, x = var_1167)[name = tensor("fill_mask_17")]; + tensor var_844 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_844")]; + tensor var_845_dtype_0 = const()[name = tensor("op_845_dtype_0"), val = tensor("fp32")]; + tensor var_846 = const()[name = tensor("op_846"), val = tensor([1, 16, 1])]; + tensor var_845 = cast(dtype = var_845_dtype_0, x = var_844)[name = tensor("cast_159")]; + tensor fill_mask_17 = reshape(shape = var_846, x = var_845)[name = tensor("fill_mask_17")]; tensor x_expanded_17_reps_0 = const()[name = tensor("x_expanded_17_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_1158)[name = tensor("x_expanded_17")]; - tensor var_1172 = sub(x = var_17, y = fill_mask_17)[name = tensor("op_1172")]; - tensor var_1173 = mul(x = window_5, y = var_1172)[name = tensor("op_1173")]; - tensor var_1174 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_1174")]; - tensor filled_window_17 = add(x = var_1173, y = var_1174)[name = tensor("filled_window_17")]; - tensor var_1176 = greater_equal(x = filled_17, y = var_16)[name = tensor("op_1176")]; - tensor var_1177_dtype_0 = const()[name = tensor("op_1177_dtype_0"), val = tensor("fp32")]; - tensor var_1178 = const()[name = tensor("op_1178"), val = tensor([1, 1, 1])]; - tensor var_1177 = cast(dtype = var_1177_dtype_0, x = var_1176)[name = tensor("cast_158")]; - tensor use_roll_17 = reshape(shape = var_1178, x = var_1177)[name = tensor("use_roll_17")]; - tensor var_1180 = sub(x = var_17, y = use_roll_17)[name = tensor("op_1180")]; - tensor var_1181 = mul(x = filled_window_17, y = var_1180)[name = tensor("op_1181")]; - tensor var_1182 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_1182")]; - tensor input_161 = add(x = var_1181, y = var_1182)[name = tensor("input_161")]; - tensor var_1184 = const()[name = tensor("op_1184"), val = tensor(0x1p+0)]; - tensor var_1185 = add(x = filled_17, y = var_1184)[name = tensor("op_1185")]; - tensor clip_17 = clip(alpha = var_21, beta = var_16, x = var_1185)[name = tensor("clip_17")]; - tensor x_29_axes_0 = const()[name = tensor("x_29_axes_0"), val = tensor([-1])]; - tensor x_29 = layer_norm(axes = x_29_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_161)[name = tensor("x_29")]; - tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_59_pad_type_0 = const()[name = tensor("inputs_59_pad_type_0"), val = tensor("valid")]; - tensor inputs_59_strides_0 = const()[name = tensor("inputs_59_strides_0"), val = tensor([1])]; - tensor inputs_59_pad_0 = const()[name = tensor("inputs_59_pad_0"), val = tensor([0, 0])]; - tensor inputs_59_dilations_0 = const()[name = tensor("inputs_59_dilations_0"), val = tensor([1])]; - tensor inputs_59_groups_0 = const()[name = tensor("inputs_59_groups_0"), val = tensor(1)]; - tensor input_163 = transpose(perm = input_163_perm_0, x = x_29)[name = tensor("transpose_50")]; - tensor inputs_59 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_59_dilations_0, groups = inputs_59_groups_0, pad = inputs_59_pad_0, pad_type = inputs_59_pad_type_0, strides = inputs_59_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_163)[name = tensor("inputs_59")]; - tensor var_1206_split_sizes_0 = const()[name = tensor("op_1206_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1206_axis_0 = const()[name = tensor("op_1206_axis_0"), val = tensor(1)]; - tensor var_1206_0, tensor var_1206_1 = split(axis = var_1206_axis_0, split_sizes = var_1206_split_sizes_0, x = inputs_59)[name = tensor("op_1206")]; - tensor var_1208 = sigmoid(x = var_1206_1)[name = tensor("op_1208")]; - tensor inputs_61 = mul(x = var_1206_0, y = var_1208)[name = tensor("inputs_61")]; - tensor outputs_aug_17_pad_type_0 = const()[name = tensor("outputs_aug_17_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_17_pad_0 = const()[name = tensor("outputs_aug_17_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_17_groups_0 = const()[name = tensor("outputs_aug_17_groups_0"), val = tensor(256)]; - tensor outputs_aug_17_strides_0 = const()[name = tensor("outputs_aug_17_strides_0"), val = tensor([1])]; - tensor outputs_aug_17_dilations_0 = const()[name = tensor("outputs_aug_17_dilations_0"), val = tensor([1])]; - tensor outputs_aug_17 = conv(dilations = outputs_aug_17_dilations_0, groups = outputs_aug_17_groups_0, pad = outputs_aug_17_pad_0, pad_type = outputs_aug_17_pad_type_0, strides = outputs_aug_17_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_61)[name = tensor("outputs_aug_17")]; - tensor input_165_begin_0 = const()[name = tensor("input_165_begin_0"), val = tensor([0, 0, 0])]; - tensor input_165_end_0 = const()[name = tensor("input_165_end_0"), val = tensor([1, 256, 16])]; - tensor input_165_end_mask_0 = const()[name = tensor("input_165_end_mask_0"), val = tensor([true, true, false])]; - tensor input_165 = slice_by_index(begin = input_165_begin_0, end = input_165_end_0, end_mask = input_165_end_mask_0, x = outputs_aug_17)[name = tensor("input_165")]; - tensor inputs_63 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_165)[name = tensor("inputs_63")]; - tensor input_167 = silu(x = inputs_63)[name = tensor("input_167")]; - tensor input_169_pad_type_0 = const()[name = tensor("input_169_pad_type_0"), val = tensor("valid")]; - tensor input_169_strides_0 = const()[name = tensor("input_169_strides_0"), val = tensor([1])]; - tensor input_169_pad_0 = const()[name = tensor("input_169_pad_0"), val = tensor([0, 0])]; - tensor input_169_dilations_0 = const()[name = tensor("input_169_dilations_0"), val = tensor([1])]; - tensor input_169_groups_0 = const()[name = tensor("input_169_groups_0"), val = tensor(1)]; - tensor input_169 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_169_dilations_0, groups = input_169_groups_0, pad = input_169_pad_0, pad_type = input_169_pad_type_0, strides = input_169_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_167)[name = tensor("input_169")]; - tensor conv_out_17_perm_0 = const()[name = tensor("conv_out_17_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1239_begin_0 = const()[name = tensor("op_1239_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1239_end_0 = const()[name = tensor("op_1239_end_0"), val = tensor([1, 16, 256])]; - tensor var_1239_end_mask_0 = const()[name = tensor("op_1239_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_17 = transpose(perm = conv_out_17_perm_0, x = input_169)[name = tensor("transpose_49")]; - tensor var_1239 = slice_by_index(begin = var_1239_begin_0, end = var_1239_end_0, end_mask = var_1239_end_mask_0, x = conv_out_17)[name = tensor("op_1239")]; - tensor var_1242_begin_0 = const()[name = tensor("op_1242_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1242_end_0 = const()[name = tensor("op_1242_end_0"), val = tensor([1, 2, 256])]; - tensor var_1242_end_mask_0 = const()[name = tensor("op_1242_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1242 = slice_by_index(begin = var_1242_begin_0, end = var_1242_end_0, end_mask = var_1242_end_mask_0, x = x_27)[name = tensor("op_1242")]; - tensor var_1245_begin_0 = const()[name = tensor("op_1245_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1245_end_0 = const()[name = tensor("op_1245_end_0"), val = tensor([1, 16, 256])]; - tensor var_1245_end_mask_0 = const()[name = tensor("op_1245_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1245 = slice_by_index(begin = var_1245_begin_0, end = var_1245_end_0, end_mask = var_1245_end_mask_0, x = input_161)[name = tensor("op_1245")]; + tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_836)[name = tensor("x_expanded_17")]; + tensor var_850 = sub(x = var_17, y = fill_mask_17)[name = tensor("op_850")]; + tensor var_851 = mul(x = window_21, y = var_850)[name = tensor("op_851")]; + tensor var_852 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_852")]; + tensor filled_window_17 = add(x = var_851, y = var_852)[name = tensor("filled_window_17")]; + tensor var_854 = greater_equal(x = filled_17, y = var_16)[name = tensor("op_854")]; + tensor var_855_dtype_0 = const()[name = tensor("op_855_dtype_0"), val = tensor("fp32")]; + tensor var_856 = const()[name = tensor("op_856"), val = tensor([1, 1, 1])]; + tensor var_855 = cast(dtype = var_855_dtype_0, x = var_854)[name = tensor("cast_158")]; + tensor use_roll_17 = reshape(shape = var_856, x = var_855)[name = tensor("use_roll_17")]; + tensor var_858 = sub(x = var_17, y = use_roll_17)[name = tensor("op_858")]; + tensor var_859 = mul(x = filled_window_17, y = var_858)[name = tensor("op_859")]; + tensor var_860 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_860")]; + tensor window_23 = add(x = var_859, y = var_860)[name = tensor("window_23")]; + tensor var_862 = const()[name = tensor("op_862"), val = tensor(0x1p+0)]; + tensor var_863 = add(x = filled_17, y = var_862)[name = tensor("op_863")]; + tensor clip_17 = clip(alpha = var_21, beta = var_16, x = var_863)[name = tensor("clip_17")]; + tensor var_866_begin_0 = const()[name = tensor("op_866_begin_0"), val = tensor([0, 1, 0])]; + tensor var_866_end_0 = const()[name = tensor("op_866_end_0"), val = tensor([1, 2, 256])]; + tensor var_866_end_mask_0 = const()[name = tensor("op_866_end_mask_0"), val = tensor([true, false, true])]; + tensor var_866 = slice_by_index(begin = var_866_begin_0, end = var_866_end_0, end_mask = var_866_end_mask_0, x = x_15)[name = tensor("op_866")]; + tensor var_869_begin_0 = const()[name = tensor("op_869_begin_0"), val = tensor([0, 1, 0])]; + tensor var_869_end_0 = const()[name = tensor("op_869_end_0"), val = tensor([1, 16, 256])]; + tensor var_869_end_mask_0 = const()[name = tensor("op_869_end_mask_0"), val = tensor([true, true, true])]; + tensor var_869 = slice_by_index(begin = var_869_begin_0, end = var_869_end_0, end_mask = var_869_end_mask_0, x = window_23)[name = tensor("op_869")]; tensor rolled_19_interleave_0 = const()[name = tensor("rolled_19_interleave_0"), val = tensor(false)]; - tensor rolled_19 = concat(axis = var_33, interleave = rolled_19_interleave_0, values = (var_1245, var_1242))[name = tensor("rolled_19")]; + tensor rolled_19 = concat(axis = var_33, interleave = rolled_19_interleave_0, values = (var_869, var_866))[name = tensor("rolled_19")]; tensor clip_18 = clip(alpha = var_21, beta = var_20, x = clip_17)[name = tensor("clip_18")]; - tensor var_1250 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_1250")]; - tensor var_1251_dtype_0 = const()[name = tensor("op_1251_dtype_0"), val = tensor("fp32")]; - tensor var_1252 = const()[name = tensor("op_1252"), val = tensor([1, 16, 1])]; - tensor var_1251 = cast(dtype = var_1251_dtype_0, x = var_1250)[name = tensor("cast_157")]; - tensor fill_mask_19 = reshape(shape = var_1252, x = var_1251)[name = tensor("fill_mask_19")]; + tensor var_874 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_874")]; + tensor var_875_dtype_0 = const()[name = tensor("op_875_dtype_0"), val = tensor("fp32")]; + tensor var_876 = const()[name = tensor("op_876"), val = tensor([1, 16, 1])]; + tensor var_875 = cast(dtype = var_875_dtype_0, x = var_874)[name = tensor("cast_157")]; + tensor fill_mask_19 = reshape(shape = var_876, x = var_875)[name = tensor("fill_mask_19")]; tensor x_expanded_19_reps_0 = const()[name = tensor("x_expanded_19_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_1242)[name = tensor("x_expanded_19")]; - tensor var_1256 = sub(x = var_17, y = fill_mask_19)[name = tensor("op_1256")]; - tensor var_1257 = mul(x = input_161, y = var_1256)[name = tensor("op_1257")]; - tensor var_1258 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_1258")]; - tensor filled_window_19 = add(x = var_1257, y = var_1258)[name = tensor("filled_window_19")]; - tensor var_1260 = greater_equal(x = clip_17, y = var_16)[name = tensor("op_1260")]; - tensor var_1261_dtype_0 = const()[name = tensor("op_1261_dtype_0"), val = tensor("fp32")]; - tensor var_1262 = const()[name = tensor("op_1262"), val = tensor([1, 1, 1])]; - tensor var_1261 = cast(dtype = var_1261_dtype_0, x = var_1260)[name = tensor("cast_156")]; - tensor use_roll_19 = reshape(shape = var_1262, x = var_1261)[name = tensor("use_roll_19")]; - tensor var_1264 = sub(x = var_17, y = use_roll_19)[name = tensor("op_1264")]; - tensor var_1265 = mul(x = filled_window_19, y = var_1264)[name = tensor("op_1265")]; - tensor var_1266 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_1266")]; - tensor input_171 = add(x = var_1265, y = var_1266)[name = tensor("input_171")]; - tensor var_1268 = const()[name = tensor("op_1268"), val = tensor(0x1p+0)]; - tensor var_1269 = add(x = clip_17, y = var_1268)[name = tensor("op_1269")]; - tensor clip_19 = clip(alpha = var_21, beta = var_16, x = var_1269)[name = tensor("clip_19")]; - tensor x_31_axes_0 = const()[name = tensor("x_31_axes_0"), val = tensor([-1])]; - tensor x_31 = layer_norm(axes = x_31_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_171)[name = tensor("x_31")]; - tensor input_173_perm_0 = const()[name = tensor("input_173_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_65_pad_type_0 = const()[name = tensor("inputs_65_pad_type_0"), val = tensor("valid")]; - tensor inputs_65_strides_0 = const()[name = tensor("inputs_65_strides_0"), val = tensor([1])]; - tensor inputs_65_pad_0 = const()[name = tensor("inputs_65_pad_0"), val = tensor([0, 0])]; - tensor inputs_65_dilations_0 = const()[name = tensor("inputs_65_dilations_0"), val = tensor([1])]; - tensor inputs_65_groups_0 = const()[name = tensor("inputs_65_groups_0"), val = tensor(1)]; - tensor input_173 = transpose(perm = input_173_perm_0, x = x_31)[name = tensor("transpose_48")]; - tensor inputs_65 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_65_dilations_0, groups = inputs_65_groups_0, pad = inputs_65_pad_0, pad_type = inputs_65_pad_type_0, strides = inputs_65_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_173)[name = tensor("inputs_65")]; - tensor var_1290_split_sizes_0 = const()[name = tensor("op_1290_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1290_axis_0 = const()[name = tensor("op_1290_axis_0"), val = tensor(1)]; - tensor var_1290_0, tensor var_1290_1 = split(axis = var_1290_axis_0, split_sizes = var_1290_split_sizes_0, x = inputs_65)[name = tensor("op_1290")]; - tensor var_1292 = sigmoid(x = var_1290_1)[name = tensor("op_1292")]; - tensor inputs_67 = mul(x = var_1290_0, y = var_1292)[name = tensor("inputs_67")]; - tensor outputs_aug_19_pad_type_0 = const()[name = tensor("outputs_aug_19_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_19_pad_0 = const()[name = tensor("outputs_aug_19_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_19_groups_0 = const()[name = tensor("outputs_aug_19_groups_0"), val = tensor(256)]; - tensor outputs_aug_19_strides_0 = const()[name = tensor("outputs_aug_19_strides_0"), val = tensor([1])]; - tensor outputs_aug_19_dilations_0 = const()[name = tensor("outputs_aug_19_dilations_0"), val = tensor([1])]; - tensor outputs_aug_19 = conv(dilations = outputs_aug_19_dilations_0, groups = outputs_aug_19_groups_0, pad = outputs_aug_19_pad_0, pad_type = outputs_aug_19_pad_type_0, strides = outputs_aug_19_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_67)[name = tensor("outputs_aug_19")]; - tensor input_175_begin_0 = const()[name = tensor("input_175_begin_0"), val = tensor([0, 0, 0])]; - tensor input_175_end_0 = const()[name = tensor("input_175_end_0"), val = tensor([1, 256, 16])]; - tensor input_175_end_mask_0 = const()[name = tensor("input_175_end_mask_0"), val = tensor([true, true, false])]; - tensor input_175 = slice_by_index(begin = input_175_begin_0, end = input_175_end_0, end_mask = input_175_end_mask_0, x = outputs_aug_19)[name = tensor("input_175")]; - tensor inputs_69 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_175)[name = tensor("inputs_69")]; - tensor input_177 = silu(x = inputs_69)[name = tensor("input_177")]; - tensor input_179_pad_type_0 = const()[name = tensor("input_179_pad_type_0"), val = tensor("valid")]; - tensor input_179_strides_0 = const()[name = tensor("input_179_strides_0"), val = tensor([1])]; - tensor input_179_pad_0 = const()[name = tensor("input_179_pad_0"), val = tensor([0, 0])]; - tensor input_179_dilations_0 = const()[name = tensor("input_179_dilations_0"), val = tensor([1])]; - tensor input_179_groups_0 = const()[name = tensor("input_179_groups_0"), val = tensor(1)]; - tensor input_179 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_179_dilations_0, groups = input_179_groups_0, pad = input_179_pad_0, pad_type = input_179_pad_type_0, strides = input_179_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_177)[name = tensor("input_179")]; - tensor conv_out_19_perm_0 = const()[name = tensor("conv_out_19_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1323_begin_0 = const()[name = tensor("op_1323_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1323_end_0 = const()[name = tensor("op_1323_end_0"), val = tensor([1, 16, 256])]; - tensor var_1323_end_mask_0 = const()[name = tensor("op_1323_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_19 = transpose(perm = conv_out_19_perm_0, x = input_179)[name = tensor("transpose_47")]; - tensor var_1323 = slice_by_index(begin = var_1323_begin_0, end = var_1323_end_0, end_mask = var_1323_end_mask_0, x = conv_out_19)[name = tensor("op_1323")]; - tensor var_1326_begin_0 = const()[name = tensor("op_1326_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1326_end_0 = const()[name = tensor("op_1326_end_0"), val = tensor([1, 3, 256])]; - tensor var_1326_end_mask_0 = const()[name = tensor("op_1326_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1326 = slice_by_index(begin = var_1326_begin_0, end = var_1326_end_0, end_mask = var_1326_end_mask_0, x = x_27)[name = tensor("op_1326")]; - tensor var_1329_begin_0 = const()[name = tensor("op_1329_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1329_end_0 = const()[name = tensor("op_1329_end_0"), val = tensor([1, 16, 256])]; - tensor var_1329_end_mask_0 = const()[name = tensor("op_1329_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1329 = slice_by_index(begin = var_1329_begin_0, end = var_1329_end_0, end_mask = var_1329_end_mask_0, x = input_171)[name = tensor("op_1329")]; + tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_866)[name = tensor("x_expanded_19")]; + tensor var_880 = sub(x = var_17, y = fill_mask_19)[name = tensor("op_880")]; + tensor var_881 = mul(x = window_23, y = var_880)[name = tensor("op_881")]; + tensor var_882 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_882")]; + tensor filled_window_19 = add(x = var_881, y = var_882)[name = tensor("filled_window_19")]; + tensor var_884 = greater_equal(x = clip_17, y = var_16)[name = tensor("op_884")]; + tensor var_885_dtype_0 = const()[name = tensor("op_885_dtype_0"), val = tensor("fp32")]; + tensor var_886 = const()[name = tensor("op_886"), val = tensor([1, 1, 1])]; + tensor var_885 = cast(dtype = var_885_dtype_0, x = var_884)[name = tensor("cast_156")]; + tensor use_roll_19 = reshape(shape = var_886, x = var_885)[name = tensor("use_roll_19")]; + tensor var_888 = sub(x = var_17, y = use_roll_19)[name = tensor("op_888")]; + tensor var_889 = mul(x = filled_window_19, y = var_888)[name = tensor("op_889")]; + tensor var_890 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_890")]; + tensor window_25 = add(x = var_889, y = var_890)[name = tensor("window_25")]; + tensor var_892 = const()[name = tensor("op_892"), val = tensor(0x1p+0)]; + tensor var_893 = add(x = clip_17, y = var_892)[name = tensor("op_893")]; + tensor clip_19 = clip(alpha = var_21, beta = var_16, x = var_893)[name = tensor("clip_19")]; + tensor var_896_begin_0 = const()[name = tensor("op_896_begin_0"), val = tensor([0, 2, 0])]; + tensor var_896_end_0 = const()[name = tensor("op_896_end_0"), val = tensor([1, 3, 256])]; + tensor var_896_end_mask_0 = const()[name = tensor("op_896_end_mask_0"), val = tensor([true, false, true])]; + tensor var_896 = slice_by_index(begin = var_896_begin_0, end = var_896_end_0, end_mask = var_896_end_mask_0, x = x_15)[name = tensor("op_896")]; + tensor var_899_begin_0 = const()[name = tensor("op_899_begin_0"), val = tensor([0, 1, 0])]; + tensor var_899_end_0 = const()[name = tensor("op_899_end_0"), val = tensor([1, 16, 256])]; + tensor var_899_end_mask_0 = const()[name = tensor("op_899_end_mask_0"), val = tensor([true, true, true])]; + tensor var_899 = slice_by_index(begin = var_899_begin_0, end = var_899_end_0, end_mask = var_899_end_mask_0, x = window_25)[name = tensor("op_899")]; tensor rolled_21_interleave_0 = const()[name = tensor("rolled_21_interleave_0"), val = tensor(false)]; - tensor rolled_21 = concat(axis = var_33, interleave = rolled_21_interleave_0, values = (var_1329, var_1326))[name = tensor("rolled_21")]; + tensor rolled_21 = concat(axis = var_33, interleave = rolled_21_interleave_0, values = (var_899, var_896))[name = tensor("rolled_21")]; tensor clip_20 = clip(alpha = var_21, beta = var_20, x = clip_19)[name = tensor("clip_20")]; - tensor var_1334 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_1334")]; - tensor var_1335_dtype_0 = const()[name = tensor("op_1335_dtype_0"), val = tensor("fp32")]; - tensor var_1336 = const()[name = tensor("op_1336"), val = tensor([1, 16, 1])]; - tensor var_1335 = cast(dtype = var_1335_dtype_0, x = var_1334)[name = tensor("cast_155")]; - tensor fill_mask_21 = reshape(shape = var_1336, x = var_1335)[name = tensor("fill_mask_21")]; + tensor var_904 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_904")]; + tensor var_905_dtype_0 = const()[name = tensor("op_905_dtype_0"), val = tensor("fp32")]; + tensor var_906 = const()[name = tensor("op_906"), val = tensor([1, 16, 1])]; + tensor var_905 = cast(dtype = var_905_dtype_0, x = var_904)[name = tensor("cast_155")]; + tensor fill_mask_21 = reshape(shape = var_906, x = var_905)[name = tensor("fill_mask_21")]; tensor x_expanded_21_reps_0 = const()[name = tensor("x_expanded_21_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_1326)[name = tensor("x_expanded_21")]; - tensor var_1340 = sub(x = var_17, y = fill_mask_21)[name = tensor("op_1340")]; - tensor var_1341 = mul(x = input_171, y = var_1340)[name = tensor("op_1341")]; - tensor var_1342 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_1342")]; - tensor filled_window_21 = add(x = var_1341, y = var_1342)[name = tensor("filled_window_21")]; - tensor var_1344 = greater_equal(x = clip_19, y = var_16)[name = tensor("op_1344")]; - tensor var_1345_dtype_0 = const()[name = tensor("op_1345_dtype_0"), val = tensor("fp32")]; - tensor var_1346 = const()[name = tensor("op_1346"), val = tensor([1, 1, 1])]; - tensor var_1345 = cast(dtype = var_1345_dtype_0, x = var_1344)[name = tensor("cast_154")]; - tensor use_roll_21 = reshape(shape = var_1346, x = var_1345)[name = tensor("use_roll_21")]; - tensor var_1348 = sub(x = var_17, y = use_roll_21)[name = tensor("op_1348")]; - tensor var_1349 = mul(x = filled_window_21, y = var_1348)[name = tensor("op_1349")]; - tensor var_1350 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_1350")]; - tensor input_181 = add(x = var_1349, y = var_1350)[name = tensor("input_181")]; - tensor var_1352 = const()[name = tensor("op_1352"), val = tensor(0x1p+0)]; - tensor var_1353 = add(x = clip_19, y = var_1352)[name = tensor("op_1353")]; - tensor clip_21 = clip(alpha = var_21, beta = var_16, x = var_1353)[name = tensor("clip_21")]; - tensor x_33_axes_0 = const()[name = tensor("x_33_axes_0"), val = tensor([-1])]; - tensor x_33 = layer_norm(axes = x_33_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_181)[name = tensor("x_33")]; - tensor input_183_perm_0 = const()[name = tensor("input_183_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_71_pad_type_0 = const()[name = tensor("inputs_71_pad_type_0"), val = tensor("valid")]; - tensor inputs_71_strides_0 = const()[name = tensor("inputs_71_strides_0"), val = tensor([1])]; - tensor inputs_71_pad_0 = const()[name = tensor("inputs_71_pad_0"), val = tensor([0, 0])]; - tensor inputs_71_dilations_0 = const()[name = tensor("inputs_71_dilations_0"), val = tensor([1])]; - tensor inputs_71_groups_0 = const()[name = tensor("inputs_71_groups_0"), val = tensor(1)]; - tensor input_183 = transpose(perm = input_183_perm_0, x = x_33)[name = tensor("transpose_46")]; - tensor inputs_71 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_71_dilations_0, groups = inputs_71_groups_0, pad = inputs_71_pad_0, pad_type = inputs_71_pad_type_0, strides = inputs_71_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_183)[name = tensor("inputs_71")]; - tensor var_1374_split_sizes_0 = const()[name = tensor("op_1374_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1374_axis_0 = const()[name = tensor("op_1374_axis_0"), val = tensor(1)]; - tensor var_1374_0, tensor var_1374_1 = split(axis = var_1374_axis_0, split_sizes = var_1374_split_sizes_0, x = inputs_71)[name = tensor("op_1374")]; - tensor var_1376 = sigmoid(x = var_1374_1)[name = tensor("op_1376")]; - tensor inputs_73 = mul(x = var_1374_0, y = var_1376)[name = tensor("inputs_73")]; - tensor outputs_aug_21_pad_type_0 = const()[name = tensor("outputs_aug_21_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_21_pad_0 = const()[name = tensor("outputs_aug_21_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_21_groups_0 = const()[name = tensor("outputs_aug_21_groups_0"), val = tensor(256)]; - tensor outputs_aug_21_strides_0 = const()[name = tensor("outputs_aug_21_strides_0"), val = tensor([1])]; - tensor outputs_aug_21_dilations_0 = const()[name = tensor("outputs_aug_21_dilations_0"), val = tensor([1])]; - tensor outputs_aug_21 = conv(dilations = outputs_aug_21_dilations_0, groups = outputs_aug_21_groups_0, pad = outputs_aug_21_pad_0, pad_type = outputs_aug_21_pad_type_0, strides = outputs_aug_21_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_73)[name = tensor("outputs_aug_21")]; - tensor input_185_begin_0 = const()[name = tensor("input_185_begin_0"), val = tensor([0, 0, 0])]; - tensor input_185_end_0 = const()[name = tensor("input_185_end_0"), val = tensor([1, 256, 16])]; - tensor input_185_end_mask_0 = const()[name = tensor("input_185_end_mask_0"), val = tensor([true, true, false])]; - tensor input_185 = slice_by_index(begin = input_185_begin_0, end = input_185_end_0, end_mask = input_185_end_mask_0, x = outputs_aug_21)[name = tensor("input_185")]; - tensor inputs_75 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_185)[name = tensor("inputs_75")]; - tensor input_187 = silu(x = inputs_75)[name = tensor("input_187")]; - tensor input_189_pad_type_0 = const()[name = tensor("input_189_pad_type_0"), val = tensor("valid")]; - tensor input_189_strides_0 = const()[name = tensor("input_189_strides_0"), val = tensor([1])]; - tensor input_189_pad_0 = const()[name = tensor("input_189_pad_0"), val = tensor([0, 0])]; - tensor input_189_dilations_0 = const()[name = tensor("input_189_dilations_0"), val = tensor([1])]; - tensor input_189_groups_0 = const()[name = tensor("input_189_groups_0"), val = tensor(1)]; - tensor input_189 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_189_dilations_0, groups = input_189_groups_0, pad = input_189_pad_0, pad_type = input_189_pad_type_0, strides = input_189_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_187)[name = tensor("input_189")]; - tensor conv_out_21_perm_0 = const()[name = tensor("conv_out_21_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1407_begin_0 = const()[name = tensor("op_1407_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1407_end_0 = const()[name = tensor("op_1407_end_0"), val = tensor([1, 16, 256])]; - tensor var_1407_end_mask_0 = const()[name = tensor("op_1407_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_21 = transpose(perm = conv_out_21_perm_0, x = input_189)[name = tensor("transpose_45")]; - tensor var_1407 = slice_by_index(begin = var_1407_begin_0, end = var_1407_end_0, end_mask = var_1407_end_mask_0, x = conv_out_21)[name = tensor("op_1407")]; - tensor var_1410_begin_0 = const()[name = tensor("op_1410_begin_0"), val = tensor([0, 3, 0])]; - tensor var_1410_end_0 = const()[name = tensor("op_1410_end_0"), val = tensor([1, 1, 256])]; - tensor var_1410_end_mask_0 = const()[name = tensor("op_1410_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1410 = slice_by_index(begin = var_1410_begin_0, end = var_1410_end_0, end_mask = var_1410_end_mask_0, x = x_27)[name = tensor("op_1410")]; - tensor var_1413_begin_0 = const()[name = tensor("op_1413_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1413_end_0 = const()[name = tensor("op_1413_end_0"), val = tensor([1, 16, 256])]; - tensor var_1413_end_mask_0 = const()[name = tensor("op_1413_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1413 = slice_by_index(begin = var_1413_begin_0, end = var_1413_end_0, end_mask = var_1413_end_mask_0, x = input_181)[name = tensor("op_1413")]; + tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_896)[name = tensor("x_expanded_21")]; + tensor var_910 = sub(x = var_17, y = fill_mask_21)[name = tensor("op_910")]; + tensor var_911 = mul(x = window_25, y = var_910)[name = tensor("op_911")]; + tensor var_912 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_912")]; + tensor filled_window_21 = add(x = var_911, y = var_912)[name = tensor("filled_window_21")]; + tensor var_914 = greater_equal(x = clip_19, y = var_16)[name = tensor("op_914")]; + tensor var_915_dtype_0 = const()[name = tensor("op_915_dtype_0"), val = tensor("fp32")]; + tensor var_916 = const()[name = tensor("op_916"), val = tensor([1, 1, 1])]; + tensor var_915 = cast(dtype = var_915_dtype_0, x = var_914)[name = tensor("cast_154")]; + tensor use_roll_21 = reshape(shape = var_916, x = var_915)[name = tensor("use_roll_21")]; + tensor var_918 = sub(x = var_17, y = use_roll_21)[name = tensor("op_918")]; + tensor var_919 = mul(x = filled_window_21, y = var_918)[name = tensor("op_919")]; + tensor var_920 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_920")]; + tensor window_27 = add(x = var_919, y = var_920)[name = tensor("window_27")]; + tensor var_922 = const()[name = tensor("op_922"), val = tensor(0x1p+0)]; + tensor var_923 = add(x = clip_19, y = var_922)[name = tensor("op_923")]; + tensor clip_21 = clip(alpha = var_21, beta = var_16, x = var_923)[name = tensor("clip_21")]; + tensor var_926_begin_0 = const()[name = tensor("op_926_begin_0"), val = tensor([0, 3, 0])]; + tensor var_926_end_0 = const()[name = tensor("op_926_end_0"), val = tensor([1, 1, 256])]; + tensor var_926_end_mask_0 = const()[name = tensor("op_926_end_mask_0"), val = tensor([true, true, true])]; + tensor var_926 = slice_by_index(begin = var_926_begin_0, end = var_926_end_0, end_mask = var_926_end_mask_0, x = x_15)[name = tensor("op_926")]; + tensor var_929_begin_0 = const()[name = tensor("op_929_begin_0"), val = tensor([0, 1, 0])]; + tensor var_929_end_0 = const()[name = tensor("op_929_end_0"), val = tensor([1, 16, 256])]; + tensor var_929_end_mask_0 = const()[name = tensor("op_929_end_mask_0"), val = tensor([true, true, true])]; + tensor var_929 = slice_by_index(begin = var_929_begin_0, end = var_929_end_0, end_mask = var_929_end_mask_0, x = window_27)[name = tensor("op_929")]; tensor rolled_23_interleave_0 = const()[name = tensor("rolled_23_interleave_0"), val = tensor(false)]; - tensor rolled_23 = concat(axis = var_33, interleave = rolled_23_interleave_0, values = (var_1413, var_1410))[name = tensor("rolled_23")]; + tensor rolled_23 = concat(axis = var_33, interleave = rolled_23_interleave_0, values = (var_929, var_926))[name = tensor("rolled_23")]; tensor clip_22 = clip(alpha = var_21, beta = var_20, x = clip_21)[name = tensor("clip_22")]; - tensor var_1418 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_1418")]; - tensor var_1419_dtype_0 = const()[name = tensor("op_1419_dtype_0"), val = tensor("fp32")]; - tensor var_1420 = const()[name = tensor("op_1420"), val = tensor([1, 16, 1])]; - tensor var_1419 = cast(dtype = var_1419_dtype_0, x = var_1418)[name = tensor("cast_153")]; - tensor fill_mask_23 = reshape(shape = var_1420, x = var_1419)[name = tensor("fill_mask_23")]; + tensor var_934 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_934")]; + tensor var_935_dtype_0 = const()[name = tensor("op_935_dtype_0"), val = tensor("fp32")]; + tensor var_936 = const()[name = tensor("op_936"), val = tensor([1, 16, 1])]; + tensor var_935 = cast(dtype = var_935_dtype_0, x = var_934)[name = tensor("cast_153")]; + tensor fill_mask_23 = reshape(shape = var_936, x = var_935)[name = tensor("fill_mask_23")]; tensor x_expanded_23_reps_0 = const()[name = tensor("x_expanded_23_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_23 = tile(reps = x_expanded_23_reps_0, x = var_1410)[name = tensor("x_expanded_23")]; - tensor var_1424 = sub(x = var_17, y = fill_mask_23)[name = tensor("op_1424")]; - tensor var_1425 = mul(x = input_181, y = var_1424)[name = tensor("op_1425")]; - tensor var_1426 = mul(x = x_expanded_23, y = fill_mask_23)[name = tensor("op_1426")]; - tensor filled_window_23 = add(x = var_1425, y = var_1426)[name = tensor("filled_window_23")]; - tensor var_1428 = greater_equal(x = clip_21, y = var_16)[name = tensor("op_1428")]; - tensor var_1429_dtype_0 = const()[name = tensor("op_1429_dtype_0"), val = tensor("fp32")]; - tensor var_1430 = const()[name = tensor("op_1430"), val = tensor([1, 1, 1])]; - tensor var_1429 = cast(dtype = var_1429_dtype_0, x = var_1428)[name = tensor("cast_152")]; - tensor use_roll_23 = reshape(shape = var_1430, x = var_1429)[name = tensor("use_roll_23")]; - tensor var_1432 = sub(x = var_17, y = use_roll_23)[name = tensor("op_1432")]; - tensor var_1433 = mul(x = filled_window_23, y = var_1432)[name = tensor("op_1433")]; - tensor var_1434 = mul(x = rolled_23, y = use_roll_23)[name = tensor("op_1434")]; - tensor input_191 = add(x = var_1433, y = var_1434)[name = tensor("input_191")]; - tensor var_1436 = const()[name = tensor("op_1436"), val = tensor(0x1p+0)]; - tensor var_1437 = add(x = clip_21, y = var_1436)[name = tensor("op_1437")]; - tensor clip_23 = clip(alpha = var_21, beta = var_16, x = var_1437)[name = tensor("clip_23")]; - tensor x_35_axes_0 = const()[name = tensor("x_35_axes_0"), val = tensor([-1])]; - tensor x_35 = layer_norm(axes = x_35_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_191)[name = tensor("x_35")]; - tensor input_193_perm_0 = const()[name = tensor("input_193_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_77_pad_type_0 = const()[name = tensor("inputs_77_pad_type_0"), val = tensor("valid")]; - tensor inputs_77_strides_0 = const()[name = tensor("inputs_77_strides_0"), val = tensor([1])]; - tensor inputs_77_pad_0 = const()[name = tensor("inputs_77_pad_0"), val = tensor([0, 0])]; - tensor inputs_77_dilations_0 = const()[name = tensor("inputs_77_dilations_0"), val = tensor([1])]; - tensor inputs_77_groups_0 = const()[name = tensor("inputs_77_groups_0"), val = tensor(1)]; - tensor input_193 = transpose(perm = input_193_perm_0, x = x_35)[name = tensor("transpose_44")]; - tensor inputs_77 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_77_dilations_0, groups = inputs_77_groups_0, pad = inputs_77_pad_0, pad_type = inputs_77_pad_type_0, strides = inputs_77_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_193)[name = tensor("inputs_77")]; - tensor var_1458_split_sizes_0 = const()[name = tensor("op_1458_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1458_axis_0 = const()[name = tensor("op_1458_axis_0"), val = tensor(1)]; - tensor var_1458_0, tensor var_1458_1 = split(axis = var_1458_axis_0, split_sizes = var_1458_split_sizes_0, x = inputs_77)[name = tensor("op_1458")]; - tensor var_1460 = sigmoid(x = var_1458_1)[name = tensor("op_1460")]; - tensor inputs_79 = mul(x = var_1458_0, y = var_1460)[name = tensor("inputs_79")]; - tensor outputs_aug_23_pad_type_0 = const()[name = tensor("outputs_aug_23_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_23_pad_0 = const()[name = tensor("outputs_aug_23_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_23_groups_0 = const()[name = tensor("outputs_aug_23_groups_0"), val = tensor(256)]; - tensor outputs_aug_23_strides_0 = const()[name = tensor("outputs_aug_23_strides_0"), val = tensor([1])]; - tensor outputs_aug_23_dilations_0 = const()[name = tensor("outputs_aug_23_dilations_0"), val = tensor([1])]; - tensor outputs_aug_23 = conv(dilations = outputs_aug_23_dilations_0, groups = outputs_aug_23_groups_0, pad = outputs_aug_23_pad_0, pad_type = outputs_aug_23_pad_type_0, strides = outputs_aug_23_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_79)[name = tensor("outputs_aug_23")]; - tensor input_195_begin_0 = const()[name = tensor("input_195_begin_0"), val = tensor([0, 0, 0])]; - tensor input_195_end_0 = const()[name = tensor("input_195_end_0"), val = tensor([1, 256, 16])]; - tensor input_195_end_mask_0 = const()[name = tensor("input_195_end_mask_0"), val = tensor([true, true, false])]; - tensor input_195 = slice_by_index(begin = input_195_begin_0, end = input_195_end_0, end_mask = input_195_end_mask_0, x = outputs_aug_23)[name = tensor("input_195")]; - tensor inputs_81 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_195)[name = tensor("inputs_81")]; - tensor input_197 = silu(x = inputs_81)[name = tensor("input_197")]; - tensor input_199_pad_type_0 = const()[name = tensor("input_199_pad_type_0"), val = tensor("valid")]; - tensor input_199_strides_0 = const()[name = tensor("input_199_strides_0"), val = tensor([1])]; - tensor input_199_pad_0 = const()[name = tensor("input_199_pad_0"), val = tensor([0, 0])]; - tensor input_199_dilations_0 = const()[name = tensor("input_199_dilations_0"), val = tensor([1])]; - tensor input_199_groups_0 = const()[name = tensor("input_199_groups_0"), val = tensor(1)]; - tensor input_199 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_199_dilations_0, groups = input_199_groups_0, pad = input_199_pad_0, pad_type = input_199_pad_type_0, strides = input_199_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_197)[name = tensor("input_199")]; - tensor conv_out_23_perm_0 = const()[name = tensor("conv_out_23_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1491_begin_0 = const()[name = tensor("op_1491_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1491_end_0 = const()[name = tensor("op_1491_end_0"), val = tensor([1, 16, 256])]; - tensor var_1491_end_mask_0 = const()[name = tensor("op_1491_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_23 = transpose(perm = conv_out_23_perm_0, x = input_199)[name = tensor("transpose_43")]; - tensor var_1491 = slice_by_index(begin = var_1491_begin_0, end = var_1491_end_0, end_mask = var_1491_end_mask_0, x = conv_out_23)[name = tensor("op_1491")]; - tensor var_1494_interleave_0 = const()[name = tensor("op_1494_interleave_0"), val = tensor(false)]; - tensor var_1494 = concat(axis = var_33, interleave = var_1494_interleave_0, values = (var_1239, var_1323, var_1407, var_1491))[name = tensor("op_1494")]; - tensor input_201 = add(x = x_27, y = var_1494)[name = tensor("input_201")]; - tensor input_203_axes_0 = const()[name = tensor("input_203_axes_0"), val = tensor([-1])]; - tensor input_203 = layer_norm(axes = input_203_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_201)[name = tensor("input_203")]; - tensor inputs_83 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_203)[name = tensor("linear_26")]; - tensor input_205 = silu(x = inputs_83)[name = tensor("input_205")]; - tensor input_209 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_205)[name = tensor("linear_27")]; - tensor var_1517 = const()[name = tensor("op_1517"), val = tensor(0x1p-1)]; - tensor var_1518 = mul(x = input_209, y = var_1517)[name = tensor("op_1518")]; - tensor input_211 = add(x = var_1518, y = input_201)[name = tensor("input_211")]; - tensor input_213_axes_0 = const()[name = tensor("input_213_axes_0"), val = tensor([-1])]; - tensor input_213 = layer_norm(axes = input_213_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_211)[name = tensor("input_213")]; - tensor input_215_axes_0 = const()[name = tensor("input_215_axes_0"), val = tensor([-1])]; - tensor input_215 = layer_norm(axes = input_215_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_213)[name = tensor("input_215")]; - tensor inputs_85 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_215)[name = tensor("linear_28")]; - tensor input_217 = silu(x = inputs_85)[name = tensor("input_217")]; - tensor input_221 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_217)[name = tensor("linear_29")]; - tensor var_1547 = const()[name = tensor("op_1547"), val = tensor(0x1p-1)]; - tensor var_1548 = mul(x = input_221, y = var_1547)[name = tensor("op_1548")]; - tensor input_223 = add(x = var_1548, y = input_213)[name = tensor("input_223")]; - tensor x_37_axes_0 = const()[name = tensor("x_37_axes_0"), val = tensor([-1])]; - tensor x_37 = layer_norm(axes = x_37_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_223)[name = tensor("x_37")]; + tensor x_expanded_23 = tile(reps = x_expanded_23_reps_0, x = var_926)[name = tensor("x_expanded_23")]; + tensor var_940 = sub(x = var_17, y = fill_mask_23)[name = tensor("op_940")]; + tensor var_941 = mul(x = window_27, y = var_940)[name = tensor("op_941")]; + tensor var_942 = mul(x = x_expanded_23, y = fill_mask_23)[name = tensor("op_942")]; + tensor filled_window_23 = add(x = var_941, y = var_942)[name = tensor("filled_window_23")]; + tensor var_944 = greater_equal(x = clip_21, y = var_16)[name = tensor("op_944")]; + tensor var_945_dtype_0 = const()[name = tensor("op_945_dtype_0"), val = tensor("fp32")]; + tensor var_946 = const()[name = tensor("op_946"), val = tensor([1, 1, 1])]; + tensor var_945 = cast(dtype = var_945_dtype_0, x = var_944)[name = tensor("cast_152")]; + tensor use_roll_23 = reshape(shape = var_946, x = var_945)[name = tensor("use_roll_23")]; + tensor var_948 = sub(x = var_17, y = use_roll_23)[name = tensor("op_948")]; + tensor var_949 = mul(x = filled_window_23, y = var_948)[name = tensor("op_949")]; + tensor var_950 = mul(x = rolled_23, y = use_roll_23)[name = tensor("op_950")]; + tensor window_29 = add(x = var_949, y = var_950)[name = tensor("window_29")]; + tensor var_952 = const()[name = tensor("op_952"), val = tensor(0x1p+0)]; + tensor var_953 = add(x = clip_21, y = var_952)[name = tensor("op_953")]; + tensor clip_23 = clip(alpha = var_21, beta = var_16, x = var_953)[name = tensor("clip_23")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_30, interleave = input_101_interleave_0, values = (window_23, window_25, window_27, window_29))[name = tensor("input_101")]; + tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; + tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; + tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_23_pad_type_0 = const()[name = tensor("inputs_23_pad_type_0"), val = tensor("valid")]; + tensor inputs_23_strides_0 = const()[name = tensor("inputs_23_strides_0"), val = tensor([1])]; + tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; + tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; + tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; + tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; + tensor var_976_split_sizes_0 = const()[name = tensor("op_976_split_sizes_0"), val = tensor([256, 256])]; + tensor var_976_axis_0 = const()[name = tensor("op_976_axis_0"), val = tensor(1)]; + tensor var_976_0, tensor var_976_1 = split(axis = var_976_axis_0, split_sizes = var_976_split_sizes_0, x = inputs_23)[name = tensor("op_976")]; + tensor var_978 = sigmoid(x = var_976_1)[name = tensor("op_978")]; + tensor inputs_25 = mul(x = var_976_0, y = var_978)[name = tensor("inputs_25")]; + tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; + tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; + tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; + tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_25)[name = tensor("outputs_aug_5")]; + tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; + tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([4, 256, 16])]; + tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; + tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_5)[name = tensor("input_105")]; + tensor inputs_27 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_105)[name = tensor("inputs_27")]; + tensor input_107 = silu(x = inputs_27)[name = tensor("input_107")]; + tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; + tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; + tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; + tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; + tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; + tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; + tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1009_begin_0 = const()[name = tensor("op_1009_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1009_end_0 = const()[name = tensor("op_1009_end_0"), val = tensor([4, 16, 256])]; + tensor var_1009_end_mask_0 = const()[name = tensor("op_1009_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_1009 = slice_by_index(begin = var_1009_begin_0, end = var_1009_end_0, end_mask = var_1009_end_mask_0, x = conv_out_5)[name = tensor("op_1009")]; + tensor var_1011_perm_0 = const()[name = tensor("op_1011_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1011 = transpose(perm = var_1011_perm_0, x = var_1009)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_1011)[name = tensor("input_111")]; + tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; + tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; + tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; + tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; + tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; + tensor var_1034 = const()[name = tensor("op_1034"), val = tensor(0x1p-1)]; + tensor var_1035 = mul(x = input_119, y = var_1034)[name = tensor("op_1035")]; + tensor input_121 = add(x = var_1035, y = input_111)[name = tensor("input_121")]; + tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; + tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; + tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; + tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; + tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; + tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; + tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; + tensor var_1064 = const()[name = tensor("op_1064"), val = tensor(0x1p-1)]; + tensor var_1065 = mul(x = input_131, y = var_1064)[name = tensor("op_1065")]; + tensor input_133 = add(x = var_1065, y = input_123)[name = tensor("input_133")]; + tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; + tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; tensor prev_kv_7_end_0 = const()[name = tensor("prev_kv_7_end_0"), val = tensor([4, 1, 4, 64, 64])]; tensor prev_kv_7_end_mask_0 = const()[name = tensor("prev_kv_7_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1422,382 +1078,267 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_1562 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_37)[name = tensor("linear_30")]; - tensor var_1563 = const()[name = tensor("op_1563"), val = tensor([1, 4, 4, 64])]; - tensor var_1564 = reshape(shape = var_1563, x = var_1562)[name = tensor("op_1564")]; + tensor var_1079 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_1080 = const()[name = tensor("op_1080"), val = tensor([1, 4, 4, 64])]; + tensor var_1081 = reshape(shape = var_1080, x = var_1079)[name = tensor("op_1081")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1568 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_37)[name = tensor("linear_31")]; - tensor var_1569 = const()[name = tensor("op_1569"), val = tensor(0x1p-3)]; - tensor var_1570 = mul(x = var_1568, y = var_1569)[name = tensor("op_1570")]; - tensor var_1571 = const()[name = tensor("op_1571"), val = tensor([1, 4, 4, 64])]; - tensor var_1572 = reshape(shape = var_1571, x = var_1570)[name = tensor("op_1572")]; + tensor var_1085 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_1086 = const()[name = tensor("op_1086"), val = tensor(0x1p-3)]; + tensor var_1087 = mul(x = var_1085, y = var_1086)[name = tensor("op_1087")]; + tensor var_1088 = const()[name = tensor("op_1088"), val = tensor([1, 4, 4, 64])]; + tensor var_1089 = reshape(shape = var_1088, x = var_1087)[name = tensor("op_1089")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1576 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_37)[name = tensor("linear_32")]; - tensor var_1577 = const()[name = tensor("op_1577"), val = tensor([1, 4, 4, 64])]; - tensor var_1578 = reshape(shape = var_1577, x = var_1576)[name = tensor("op_1578")]; + tensor var_1093 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_1094 = const()[name = tensor("op_1094"), val = tensor([1, 4, 4, 64])]; + tensor var_1095 = reshape(shape = var_1094, x = var_1093)[name = tensor("op_1095")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_227 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_37)[name = tensor("linear_33")]; + tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; tensor s_t_7 = add(x = prev_scale_7, y = encoder__t_index)[name = tensor("s_t_7")]; tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_1572)[name = tensor("transpose_41")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_1564)[name = tensor("transpose_42")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_1089)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_1081)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_1588 = const()[name = tensor("op_1588"), val = tensor([4, 1])]; - tensor var_1589 = reshape(shape = var_1588, x = sqrt_s_t_7)[name = tensor("op_1589")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_1589)[name = tensor("M_7")]; - tensor var_1591 = mul(x = qk_7, y = M_7)[name = tensor("op_1591")]; + tensor var_1105 = const()[name = tensor("op_1105"), val = tensor([4, 1])]; + tensor var_1106 = reshape(shape = var_1105, x = sqrt_s_t_7)[name = tensor("op_1106")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_1106)[name = tensor("M_7")]; + tensor var_1108 = mul(x = qk_7, y = M_7)[name = tensor("op_1108")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_1578)[name = tensor("transpose_40")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1591, y = v_7)[name = tensor("inner_7")]; - tensor var_1593_transpose_x_0 = const()[name = tensor("op_1593_transpose_x_0"), val = tensor(false)]; - tensor var_1593_transpose_y_0 = const()[name = tensor("op_1593_transpose_y_0"), val = tensor(false)]; - tensor var_1593 = matmul(transpose_x = var_1593_transpose_x_0, transpose_y = var_1593_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1593")]; - tensor var_1594 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1594")]; - tensor var_1595 = const()[name = tensor("op_1595"), val = tensor([1, 1, 4, 1])]; - tensor var_1596 = reshape(shape = var_1595, x = var_1594)[name = tensor("op_1596")]; - tensor cross_7 = mul(x = var_1593, y = var_1596)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_1095)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1108, y = v_7)[name = tensor("inner_7")]; + tensor var_1110_transpose_x_0 = const()[name = tensor("op_1110_transpose_x_0"), val = tensor(false)]; + tensor var_1110_transpose_y_0 = const()[name = tensor("op_1110_transpose_y_0"), val = tensor(false)]; + tensor var_1110 = matmul(transpose_x = var_1110_transpose_x_0, transpose_y = var_1110_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1110")]; + tensor var_1111 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1111")]; + tensor var_1112 = const()[name = tensor("op_1112"), val = tensor([1, 1, 4, 1])]; + tensor var_1113 = reshape(shape = var_1112, x = var_1111)[name = tensor("op_1113")]; + tensor cross_7 = mul(x = var_1110, y = var_1113)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_1599 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1599")]; - tensor var_1601_transpose_x_1 = const()[name = tensor("op_1601_transpose_x_1"), val = tensor(true)]; - tensor var_1601_transpose_y_1 = const()[name = tensor("op_1601_transpose_y_1"), val = tensor(false)]; - tensor var_1601 = matmul(transpose_x = var_1601_transpose_x_1, transpose_y = var_1601_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1601")]; - tensor new_kv_unnorm_7 = add(x = var_1599, y = var_1601)[name = tensor("new_kv_unnorm_7")]; - tensor var_1603 = const()[name = tensor("op_1603"), val = tensor(0x1p+2)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_1603)[name = tensor("new_scale_7")]; - tensor var_1605 = sqrt(x = new_scale_7)[name = tensor("op_1605")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1605)[name = tensor("nkv_1")]; - tensor var_1607_perm_0 = const()[name = tensor("op_1607_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1116 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1116")]; + tensor var_1118_transpose_x_1 = const()[name = tensor("op_1118_transpose_x_1"), val = tensor(true)]; + tensor var_1118_transpose_y_1 = const()[name = tensor("op_1118_transpose_y_1"), val = tensor(false)]; + tensor var_1118 = matmul(transpose_x = var_1118_transpose_x_1, transpose_y = var_1118_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1118")]; + tensor new_kv_unnorm_7 = add(x = var_1116, y = var_1118)[name = tensor("new_kv_unnorm_7")]; + tensor var_1120 = const()[name = tensor("op_1120"), val = tensor(0x1p+2)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_1120)[name = tensor("new_scale_7")]; + tensor var_1122 = sqrt(x = new_scale_7)[name = tensor("op_1122")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1122)[name = tensor("nkv_1")]; + tensor var_1124_perm_0 = const()[name = tensor("op_1124_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_1607 = transpose(perm = var_1607_perm_0, x = out_19)[name = tensor("transpose_39")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1607)[name = tensor("out_21")]; - tensor var_1611 = const()[name = tensor("op_1611"), val = tensor([1, 4, 256])]; - tensor out_23 = reshape(shape = var_1611, x = out_21)[name = tensor("out_23")]; - tensor var_1613 = silu(x = input_227)[name = tensor("op_1613")]; - tensor input_229 = mul(x = var_1613, y = out_23)[name = tensor("input_229")]; - tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_229)[name = tensor("linear_34")]; - tensor x_39 = add(x = input_223, y = ret_out_7)[name = tensor("x_39")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor var_1124 = transpose(perm = var_1124_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1124)[name = tensor("out_21")]; + tensor var_1128 = const()[name = tensor("op_1128"), val = tensor([1, 4, 256])]; + tensor out_23 = reshape(shape = var_1128, x = out_21)[name = tensor("out_23")]; + tensor var_1130 = silu(x = input_137)[name = tensor("op_1130")]; + tensor input_139 = mul(x = var_1130, y = out_23)[name = tensor("input_139")]; + tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; + tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; + tensor window_31_begin_0 = const()[name = tensor("window_31_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_31_end_0 = const()[name = tensor("window_31_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_31_end_mask_0 = const()[name = tensor("window_31_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_31_squeeze_mask_0 = const()[name = tensor("window_31_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_31 = slice_by_index(begin = window_31_begin_0, end = window_31_end_0, end_mask = window_31_end_mask_0, squeeze_mask = window_31_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_31")]; tensor filled_25_begin_0 = const()[name = tensor("filled_25_begin_0"), val = tensor([3, 0])]; tensor filled_25_end_0 = const()[name = tensor("filled_25_end_0"), val = tensor([4, 1])]; tensor filled_25_end_mask_0 = const()[name = tensor("filled_25_end_mask_0"), val = tensor([false, true])]; tensor filled_25_squeeze_mask_0 = const()[name = tensor("filled_25_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_25 = slice_by_index(begin = filled_25_begin_0, end = filled_25_end_0, end_mask = filled_25_end_mask_0, squeeze_mask = filled_25_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_25")]; - tensor var_1622_begin_0 = const()[name = tensor("op_1622_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1622_end_0 = const()[name = tensor("op_1622_end_0"), val = tensor([1, 1, 256])]; - tensor var_1622_end_mask_0 = const()[name = tensor("op_1622_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1622 = slice_by_index(begin = var_1622_begin_0, end = var_1622_end_0, end_mask = var_1622_end_mask_0, x = x_39)[name = tensor("op_1622")]; - tensor var_1625_begin_0 = const()[name = tensor("op_1625_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1625_end_0 = const()[name = tensor("op_1625_end_0"), val = tensor([1, 16, 256])]; - tensor var_1625_end_mask_0 = const()[name = tensor("op_1625_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1625 = slice_by_index(begin = var_1625_begin_0, end = var_1625_end_0, end_mask = var_1625_end_mask_0, x = window)[name = tensor("op_1625")]; + tensor var_1139_begin_0 = const()[name = tensor("op_1139_begin_0"), val = tensor([0, 0, 0])]; + tensor var_1139_end_0 = const()[name = tensor("op_1139_end_0"), val = tensor([1, 1, 256])]; + tensor var_1139_end_mask_0 = const()[name = tensor("op_1139_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1139 = slice_by_index(begin = var_1139_begin_0, end = var_1139_end_0, end_mask = var_1139_end_mask_0, x = x_21)[name = tensor("op_1139")]; + tensor var_1142_begin_0 = const()[name = tensor("op_1142_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1142_end_0 = const()[name = tensor("op_1142_end_0"), val = tensor([1, 16, 256])]; + tensor var_1142_end_mask_0 = const()[name = tensor("op_1142_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1142 = slice_by_index(begin = var_1142_begin_0, end = var_1142_end_0, end_mask = var_1142_end_mask_0, x = window_31)[name = tensor("op_1142")]; tensor rolled_25_interleave_0 = const()[name = tensor("rolled_25_interleave_0"), val = tensor(false)]; - tensor rolled_25 = concat(axis = var_33, interleave = rolled_25_interleave_0, values = (var_1625, var_1622))[name = tensor("rolled_25")]; + tensor rolled_25 = concat(axis = var_33, interleave = rolled_25_interleave_0, values = (var_1142, var_1139))[name = tensor("rolled_25")]; tensor clip_24 = clip(alpha = var_21, beta = var_20, x = filled_25)[name = tensor("clip_24")]; - tensor var_1630 = equal(x = encoder__conv_positions, y = clip_24)[name = tensor("op_1630")]; - tensor var_1631_dtype_0 = const()[name = tensor("op_1631_dtype_0"), val = tensor("fp32")]; - tensor var_1632 = const()[name = tensor("op_1632"), val = tensor([1, 16, 1])]; - tensor var_1631 = cast(dtype = var_1631_dtype_0, x = var_1630)[name = tensor("cast_151")]; - tensor fill_mask_25 = reshape(shape = var_1632, x = var_1631)[name = tensor("fill_mask_25")]; + tensor var_1147 = equal(x = encoder__conv_positions, y = clip_24)[name = tensor("op_1147")]; + tensor var_1148_dtype_0 = const()[name = tensor("op_1148_dtype_0"), val = tensor("fp32")]; + tensor var_1149 = const()[name = tensor("op_1149"), val = tensor([1, 16, 1])]; + tensor var_1148 = cast(dtype = var_1148_dtype_0, x = var_1147)[name = tensor("cast_151")]; + tensor fill_mask_25 = reshape(shape = var_1149, x = var_1148)[name = tensor("fill_mask_25")]; tensor x_expanded_25_reps_0 = const()[name = tensor("x_expanded_25_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_25 = tile(reps = x_expanded_25_reps_0, x = var_1622)[name = tensor("x_expanded_25")]; - tensor var_1636 = sub(x = var_17, y = fill_mask_25)[name = tensor("op_1636")]; - tensor var_1637 = mul(x = window, y = var_1636)[name = tensor("op_1637")]; - tensor var_1638 = mul(x = x_expanded_25, y = fill_mask_25)[name = tensor("op_1638")]; - tensor filled_window_25 = add(x = var_1637, y = var_1638)[name = tensor("filled_window_25")]; - tensor var_1640 = greater_equal(x = filled_25, y = var_16)[name = tensor("op_1640")]; - tensor var_1641_dtype_0 = const()[name = tensor("op_1641_dtype_0"), val = tensor("fp32")]; - tensor var_1642 = const()[name = tensor("op_1642"), val = tensor([1, 1, 1])]; - tensor var_1641 = cast(dtype = var_1641_dtype_0, x = var_1640)[name = tensor("cast_150")]; - tensor use_roll_25 = reshape(shape = var_1642, x = var_1641)[name = tensor("use_roll_25")]; - tensor var_1644 = sub(x = var_17, y = use_roll_25)[name = tensor("op_1644")]; - tensor var_1645 = mul(x = filled_window_25, y = var_1644)[name = tensor("op_1645")]; - tensor var_1646 = mul(x = rolled_25, y = use_roll_25)[name = tensor("op_1646")]; - tensor input_231 = add(x = var_1645, y = var_1646)[name = tensor("input_231")]; - tensor var_1648 = const()[name = tensor("op_1648"), val = tensor(0x1p+0)]; - tensor var_1649 = add(x = filled_25, y = var_1648)[name = tensor("op_1649")]; - tensor clip_25 = clip(alpha = var_21, beta = var_16, x = var_1649)[name = tensor("clip_25")]; - tensor x_41_axes_0 = const()[name = tensor("x_41_axes_0"), val = tensor([-1])]; - tensor x_41 = layer_norm(axes = x_41_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_231)[name = tensor("x_41")]; - tensor input_233_perm_0 = const()[name = tensor("input_233_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_87_pad_type_0 = const()[name = tensor("inputs_87_pad_type_0"), val = tensor("valid")]; - tensor inputs_87_strides_0 = const()[name = tensor("inputs_87_strides_0"), val = tensor([1])]; - tensor inputs_87_pad_0 = const()[name = tensor("inputs_87_pad_0"), val = tensor([0, 0])]; - tensor inputs_87_dilations_0 = const()[name = tensor("inputs_87_dilations_0"), val = tensor([1])]; - tensor inputs_87_groups_0 = const()[name = tensor("inputs_87_groups_0"), val = tensor(1)]; - tensor input_233 = transpose(perm = input_233_perm_0, x = x_41)[name = tensor("transpose_38")]; - tensor inputs_87 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_87_dilations_0, groups = inputs_87_groups_0, pad = inputs_87_pad_0, pad_type = inputs_87_pad_type_0, strides = inputs_87_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_233)[name = tensor("inputs_87")]; - tensor var_1670_split_sizes_0 = const()[name = tensor("op_1670_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1670_axis_0 = const()[name = tensor("op_1670_axis_0"), val = tensor(1)]; - tensor var_1670_0, tensor var_1670_1 = split(axis = var_1670_axis_0, split_sizes = var_1670_split_sizes_0, x = inputs_87)[name = tensor("op_1670")]; - tensor var_1672 = sigmoid(x = var_1670_1)[name = tensor("op_1672")]; - tensor inputs_89 = mul(x = var_1670_0, y = var_1672)[name = tensor("inputs_89")]; - tensor outputs_aug_25_pad_type_0 = const()[name = tensor("outputs_aug_25_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_25_pad_0 = const()[name = tensor("outputs_aug_25_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_25_groups_0 = const()[name = tensor("outputs_aug_25_groups_0"), val = tensor(256)]; - tensor outputs_aug_25_strides_0 = const()[name = tensor("outputs_aug_25_strides_0"), val = tensor([1])]; - tensor outputs_aug_25_dilations_0 = const()[name = tensor("outputs_aug_25_dilations_0"), val = tensor([1])]; - tensor outputs_aug_25 = conv(dilations = outputs_aug_25_dilations_0, groups = outputs_aug_25_groups_0, pad = outputs_aug_25_pad_0, pad_type = outputs_aug_25_pad_type_0, strides = outputs_aug_25_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_89)[name = tensor("outputs_aug_25")]; - tensor input_235_begin_0 = const()[name = tensor("input_235_begin_0"), val = tensor([0, 0, 0])]; - tensor input_235_end_0 = const()[name = tensor("input_235_end_0"), val = tensor([1, 256, 16])]; - tensor input_235_end_mask_0 = const()[name = tensor("input_235_end_mask_0"), val = tensor([true, true, false])]; - tensor input_235 = slice_by_index(begin = input_235_begin_0, end = input_235_end_0, end_mask = input_235_end_mask_0, x = outputs_aug_25)[name = tensor("input_235")]; - tensor inputs_91 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_235)[name = tensor("inputs_91")]; - tensor input_237 = silu(x = inputs_91)[name = tensor("input_237")]; - tensor input_239_pad_type_0 = const()[name = tensor("input_239_pad_type_0"), val = tensor("valid")]; - tensor input_239_strides_0 = const()[name = tensor("input_239_strides_0"), val = tensor([1])]; - tensor input_239_pad_0 = const()[name = tensor("input_239_pad_0"), val = tensor([0, 0])]; - tensor input_239_dilations_0 = const()[name = tensor("input_239_dilations_0"), val = tensor([1])]; - tensor input_239_groups_0 = const()[name = tensor("input_239_groups_0"), val = tensor(1)]; - tensor input_239 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_239_dilations_0, groups = input_239_groups_0, pad = input_239_pad_0, pad_type = input_239_pad_type_0, strides = input_239_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_237)[name = tensor("input_239")]; - tensor conv_out_25_perm_0 = const()[name = tensor("conv_out_25_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1703_begin_0 = const()[name = tensor("op_1703_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1703_end_0 = const()[name = tensor("op_1703_end_0"), val = tensor([1, 16, 256])]; - tensor var_1703_end_mask_0 = const()[name = tensor("op_1703_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_25 = transpose(perm = conv_out_25_perm_0, x = input_239)[name = tensor("transpose_37")]; - tensor var_1703 = slice_by_index(begin = var_1703_begin_0, end = var_1703_end_0, end_mask = var_1703_end_mask_0, x = conv_out_25)[name = tensor("op_1703")]; - tensor var_1706_begin_0 = const()[name = tensor("op_1706_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1706_end_0 = const()[name = tensor("op_1706_end_0"), val = tensor([1, 2, 256])]; - tensor var_1706_end_mask_0 = const()[name = tensor("op_1706_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1706 = slice_by_index(begin = var_1706_begin_0, end = var_1706_end_0, end_mask = var_1706_end_mask_0, x = x_39)[name = tensor("op_1706")]; - tensor var_1709_begin_0 = const()[name = tensor("op_1709_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1709_end_0 = const()[name = tensor("op_1709_end_0"), val = tensor([1, 16, 256])]; - tensor var_1709_end_mask_0 = const()[name = tensor("op_1709_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1709 = slice_by_index(begin = var_1709_begin_0, end = var_1709_end_0, end_mask = var_1709_end_mask_0, x = input_231)[name = tensor("op_1709")]; + tensor x_expanded_25 = tile(reps = x_expanded_25_reps_0, x = var_1139)[name = tensor("x_expanded_25")]; + tensor var_1153 = sub(x = var_17, y = fill_mask_25)[name = tensor("op_1153")]; + tensor var_1154 = mul(x = window_31, y = var_1153)[name = tensor("op_1154")]; + tensor var_1155 = mul(x = x_expanded_25, y = fill_mask_25)[name = tensor("op_1155")]; + tensor filled_window_25 = add(x = var_1154, y = var_1155)[name = tensor("filled_window_25")]; + tensor var_1157 = greater_equal(x = filled_25, y = var_16)[name = tensor("op_1157")]; + tensor var_1158_dtype_0 = const()[name = tensor("op_1158_dtype_0"), val = tensor("fp32")]; + tensor var_1159 = const()[name = tensor("op_1159"), val = tensor([1, 1, 1])]; + tensor var_1158 = cast(dtype = var_1158_dtype_0, x = var_1157)[name = tensor("cast_150")]; + tensor use_roll_25 = reshape(shape = var_1159, x = var_1158)[name = tensor("use_roll_25")]; + tensor var_1161 = sub(x = var_17, y = use_roll_25)[name = tensor("op_1161")]; + tensor var_1162 = mul(x = filled_window_25, y = var_1161)[name = tensor("op_1162")]; + tensor var_1163 = mul(x = rolled_25, y = use_roll_25)[name = tensor("op_1163")]; + tensor window_33 = add(x = var_1162, y = var_1163)[name = tensor("window_33")]; + tensor var_1165 = const()[name = tensor("op_1165"), val = tensor(0x1p+0)]; + tensor var_1166 = add(x = filled_25, y = var_1165)[name = tensor("op_1166")]; + tensor clip_25 = clip(alpha = var_21, beta = var_16, x = var_1166)[name = tensor("clip_25")]; + tensor var_1169_begin_0 = const()[name = tensor("op_1169_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1169_end_0 = const()[name = tensor("op_1169_end_0"), val = tensor([1, 2, 256])]; + tensor var_1169_end_mask_0 = const()[name = tensor("op_1169_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1169 = slice_by_index(begin = var_1169_begin_0, end = var_1169_end_0, end_mask = var_1169_end_mask_0, x = x_21)[name = tensor("op_1169")]; + tensor var_1172_begin_0 = const()[name = tensor("op_1172_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1172_end_0 = const()[name = tensor("op_1172_end_0"), val = tensor([1, 16, 256])]; + tensor var_1172_end_mask_0 = const()[name = tensor("op_1172_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1172 = slice_by_index(begin = var_1172_begin_0, end = var_1172_end_0, end_mask = var_1172_end_mask_0, x = window_33)[name = tensor("op_1172")]; tensor rolled_27_interleave_0 = const()[name = tensor("rolled_27_interleave_0"), val = tensor(false)]; - tensor rolled_27 = concat(axis = var_33, interleave = rolled_27_interleave_0, values = (var_1709, var_1706))[name = tensor("rolled_27")]; + tensor rolled_27 = concat(axis = var_33, interleave = rolled_27_interleave_0, values = (var_1172, var_1169))[name = tensor("rolled_27")]; tensor clip_26 = clip(alpha = var_21, beta = var_20, x = clip_25)[name = tensor("clip_26")]; - tensor var_1714 = equal(x = encoder__conv_positions, y = clip_26)[name = tensor("op_1714")]; - tensor var_1715_dtype_0 = const()[name = tensor("op_1715_dtype_0"), val = tensor("fp32")]; - tensor var_1716 = const()[name = tensor("op_1716"), val = tensor([1, 16, 1])]; - tensor var_1715 = cast(dtype = var_1715_dtype_0, x = var_1714)[name = tensor("cast_149")]; - tensor fill_mask_27 = reshape(shape = var_1716, x = var_1715)[name = tensor("fill_mask_27")]; + tensor var_1177 = equal(x = encoder__conv_positions, y = clip_26)[name = tensor("op_1177")]; + tensor var_1178_dtype_0 = const()[name = tensor("op_1178_dtype_0"), val = tensor("fp32")]; + tensor var_1179 = const()[name = tensor("op_1179"), val = tensor([1, 16, 1])]; + tensor var_1178 = cast(dtype = var_1178_dtype_0, x = var_1177)[name = tensor("cast_149")]; + tensor fill_mask_27 = reshape(shape = var_1179, x = var_1178)[name = tensor("fill_mask_27")]; tensor x_expanded_27_reps_0 = const()[name = tensor("x_expanded_27_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_27 = tile(reps = x_expanded_27_reps_0, x = var_1706)[name = tensor("x_expanded_27")]; - tensor var_1720 = sub(x = var_17, y = fill_mask_27)[name = tensor("op_1720")]; - tensor var_1721 = mul(x = input_231, y = var_1720)[name = tensor("op_1721")]; - tensor var_1722 = mul(x = x_expanded_27, y = fill_mask_27)[name = tensor("op_1722")]; - tensor filled_window_27 = add(x = var_1721, y = var_1722)[name = tensor("filled_window_27")]; - tensor var_1724 = greater_equal(x = clip_25, y = var_16)[name = tensor("op_1724")]; - tensor var_1725_dtype_0 = const()[name = tensor("op_1725_dtype_0"), val = tensor("fp32")]; - tensor var_1726 = const()[name = tensor("op_1726"), val = tensor([1, 1, 1])]; - tensor var_1725 = cast(dtype = var_1725_dtype_0, x = var_1724)[name = tensor("cast_148")]; - tensor use_roll_27 = reshape(shape = var_1726, x = var_1725)[name = tensor("use_roll_27")]; - tensor var_1728 = sub(x = var_17, y = use_roll_27)[name = tensor("op_1728")]; - tensor var_1729 = mul(x = filled_window_27, y = var_1728)[name = tensor("op_1729")]; - tensor var_1730 = mul(x = rolled_27, y = use_roll_27)[name = tensor("op_1730")]; - tensor input_241 = add(x = var_1729, y = var_1730)[name = tensor("input_241")]; - tensor var_1732 = const()[name = tensor("op_1732"), val = tensor(0x1p+0)]; - tensor var_1733 = add(x = clip_25, y = var_1732)[name = tensor("op_1733")]; - tensor clip_27 = clip(alpha = var_21, beta = var_16, x = var_1733)[name = tensor("clip_27")]; - tensor x_43_axes_0 = const()[name = tensor("x_43_axes_0"), val = tensor([-1])]; - tensor x_43 = layer_norm(axes = x_43_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_241)[name = tensor("x_43")]; - tensor input_243_perm_0 = const()[name = tensor("input_243_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_93_pad_type_0 = const()[name = tensor("inputs_93_pad_type_0"), val = tensor("valid")]; - tensor inputs_93_strides_0 = const()[name = tensor("inputs_93_strides_0"), val = tensor([1])]; - tensor inputs_93_pad_0 = const()[name = tensor("inputs_93_pad_0"), val = tensor([0, 0])]; - tensor inputs_93_dilations_0 = const()[name = tensor("inputs_93_dilations_0"), val = tensor([1])]; - tensor inputs_93_groups_0 = const()[name = tensor("inputs_93_groups_0"), val = tensor(1)]; - tensor input_243 = transpose(perm = input_243_perm_0, x = x_43)[name = tensor("transpose_36")]; - tensor inputs_93 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_93_dilations_0, groups = inputs_93_groups_0, pad = inputs_93_pad_0, pad_type = inputs_93_pad_type_0, strides = inputs_93_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_243)[name = tensor("inputs_93")]; - tensor var_1754_split_sizes_0 = const()[name = tensor("op_1754_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1754_axis_0 = const()[name = tensor("op_1754_axis_0"), val = tensor(1)]; - tensor var_1754_0, tensor var_1754_1 = split(axis = var_1754_axis_0, split_sizes = var_1754_split_sizes_0, x = inputs_93)[name = tensor("op_1754")]; - tensor var_1756 = sigmoid(x = var_1754_1)[name = tensor("op_1756")]; - tensor inputs_95 = mul(x = var_1754_0, y = var_1756)[name = tensor("inputs_95")]; - tensor outputs_aug_27_pad_type_0 = const()[name = tensor("outputs_aug_27_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_27_pad_0 = const()[name = tensor("outputs_aug_27_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_27_groups_0 = const()[name = tensor("outputs_aug_27_groups_0"), val = tensor(256)]; - tensor outputs_aug_27_strides_0 = const()[name = tensor("outputs_aug_27_strides_0"), val = tensor([1])]; - tensor outputs_aug_27_dilations_0 = const()[name = tensor("outputs_aug_27_dilations_0"), val = tensor([1])]; - tensor outputs_aug_27 = conv(dilations = outputs_aug_27_dilations_0, groups = outputs_aug_27_groups_0, pad = outputs_aug_27_pad_0, pad_type = outputs_aug_27_pad_type_0, strides = outputs_aug_27_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_95)[name = tensor("outputs_aug_27")]; - tensor input_245_begin_0 = const()[name = tensor("input_245_begin_0"), val = tensor([0, 0, 0])]; - tensor input_245_end_0 = const()[name = tensor("input_245_end_0"), val = tensor([1, 256, 16])]; - tensor input_245_end_mask_0 = const()[name = tensor("input_245_end_mask_0"), val = tensor([true, true, false])]; - tensor input_245 = slice_by_index(begin = input_245_begin_0, end = input_245_end_0, end_mask = input_245_end_mask_0, x = outputs_aug_27)[name = tensor("input_245")]; - tensor inputs_97 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_245)[name = tensor("inputs_97")]; - tensor input_247 = silu(x = inputs_97)[name = tensor("input_247")]; - tensor input_249_pad_type_0 = const()[name = tensor("input_249_pad_type_0"), val = tensor("valid")]; - tensor input_249_strides_0 = const()[name = tensor("input_249_strides_0"), val = tensor([1])]; - tensor input_249_pad_0 = const()[name = tensor("input_249_pad_0"), val = tensor([0, 0])]; - tensor input_249_dilations_0 = const()[name = tensor("input_249_dilations_0"), val = tensor([1])]; - tensor input_249_groups_0 = const()[name = tensor("input_249_groups_0"), val = tensor(1)]; - tensor input_249 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_249_dilations_0, groups = input_249_groups_0, pad = input_249_pad_0, pad_type = input_249_pad_type_0, strides = input_249_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_247)[name = tensor("input_249")]; - tensor conv_out_27_perm_0 = const()[name = tensor("conv_out_27_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1787_begin_0 = const()[name = tensor("op_1787_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1787_end_0 = const()[name = tensor("op_1787_end_0"), val = tensor([1, 16, 256])]; - tensor var_1787_end_mask_0 = const()[name = tensor("op_1787_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_27 = transpose(perm = conv_out_27_perm_0, x = input_249)[name = tensor("transpose_35")]; - tensor var_1787 = slice_by_index(begin = var_1787_begin_0, end = var_1787_end_0, end_mask = var_1787_end_mask_0, x = conv_out_27)[name = tensor("op_1787")]; - tensor var_1790_begin_0 = const()[name = tensor("op_1790_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1790_end_0 = const()[name = tensor("op_1790_end_0"), val = tensor([1, 3, 256])]; - tensor var_1790_end_mask_0 = const()[name = tensor("op_1790_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1790 = slice_by_index(begin = var_1790_begin_0, end = var_1790_end_0, end_mask = var_1790_end_mask_0, x = x_39)[name = tensor("op_1790")]; - tensor var_1793_begin_0 = const()[name = tensor("op_1793_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1793_end_0 = const()[name = tensor("op_1793_end_0"), val = tensor([1, 16, 256])]; - tensor var_1793_end_mask_0 = const()[name = tensor("op_1793_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1793 = slice_by_index(begin = var_1793_begin_0, end = var_1793_end_0, end_mask = var_1793_end_mask_0, x = input_241)[name = tensor("op_1793")]; + tensor x_expanded_27 = tile(reps = x_expanded_27_reps_0, x = var_1169)[name = tensor("x_expanded_27")]; + tensor var_1183 = sub(x = var_17, y = fill_mask_27)[name = tensor("op_1183")]; + tensor var_1184 = mul(x = window_33, y = var_1183)[name = tensor("op_1184")]; + tensor var_1185 = mul(x = x_expanded_27, y = fill_mask_27)[name = tensor("op_1185")]; + tensor filled_window_27 = add(x = var_1184, y = var_1185)[name = tensor("filled_window_27")]; + tensor var_1187 = greater_equal(x = clip_25, y = var_16)[name = tensor("op_1187")]; + tensor var_1188_dtype_0 = const()[name = tensor("op_1188_dtype_0"), val = tensor("fp32")]; + tensor var_1189 = const()[name = tensor("op_1189"), val = tensor([1, 1, 1])]; + tensor var_1188 = cast(dtype = var_1188_dtype_0, x = var_1187)[name = tensor("cast_148")]; + tensor use_roll_27 = reshape(shape = var_1189, x = var_1188)[name = tensor("use_roll_27")]; + tensor var_1191 = sub(x = var_17, y = use_roll_27)[name = tensor("op_1191")]; + tensor var_1192 = mul(x = filled_window_27, y = var_1191)[name = tensor("op_1192")]; + tensor var_1193 = mul(x = rolled_27, y = use_roll_27)[name = tensor("op_1193")]; + tensor window_35 = add(x = var_1192, y = var_1193)[name = tensor("window_35")]; + tensor var_1195 = const()[name = tensor("op_1195"), val = tensor(0x1p+0)]; + tensor var_1196 = add(x = clip_25, y = var_1195)[name = tensor("op_1196")]; + tensor clip_27 = clip(alpha = var_21, beta = var_16, x = var_1196)[name = tensor("clip_27")]; + tensor var_1199_begin_0 = const()[name = tensor("op_1199_begin_0"), val = tensor([0, 2, 0])]; + tensor var_1199_end_0 = const()[name = tensor("op_1199_end_0"), val = tensor([1, 3, 256])]; + tensor var_1199_end_mask_0 = const()[name = tensor("op_1199_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1199 = slice_by_index(begin = var_1199_begin_0, end = var_1199_end_0, end_mask = var_1199_end_mask_0, x = x_21)[name = tensor("op_1199")]; + tensor var_1202_begin_0 = const()[name = tensor("op_1202_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1202_end_0 = const()[name = tensor("op_1202_end_0"), val = tensor([1, 16, 256])]; + tensor var_1202_end_mask_0 = const()[name = tensor("op_1202_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1202 = slice_by_index(begin = var_1202_begin_0, end = var_1202_end_0, end_mask = var_1202_end_mask_0, x = window_35)[name = tensor("op_1202")]; tensor rolled_29_interleave_0 = const()[name = tensor("rolled_29_interleave_0"), val = tensor(false)]; - tensor rolled_29 = concat(axis = var_33, interleave = rolled_29_interleave_0, values = (var_1793, var_1790))[name = tensor("rolled_29")]; + tensor rolled_29 = concat(axis = var_33, interleave = rolled_29_interleave_0, values = (var_1202, var_1199))[name = tensor("rolled_29")]; tensor clip_28 = clip(alpha = var_21, beta = var_20, x = clip_27)[name = tensor("clip_28")]; - tensor var_1798 = equal(x = encoder__conv_positions, y = clip_28)[name = tensor("op_1798")]; - tensor var_1799_dtype_0 = const()[name = tensor("op_1799_dtype_0"), val = tensor("fp32")]; - tensor var_1800 = const()[name = tensor("op_1800"), val = tensor([1, 16, 1])]; - tensor var_1799 = cast(dtype = var_1799_dtype_0, x = var_1798)[name = tensor("cast_147")]; - tensor fill_mask_29 = reshape(shape = var_1800, x = var_1799)[name = tensor("fill_mask_29")]; + tensor var_1207 = equal(x = encoder__conv_positions, y = clip_28)[name = tensor("op_1207")]; + tensor var_1208_dtype_0 = const()[name = tensor("op_1208_dtype_0"), val = tensor("fp32")]; + tensor var_1209 = const()[name = tensor("op_1209"), val = tensor([1, 16, 1])]; + tensor var_1208 = cast(dtype = var_1208_dtype_0, x = var_1207)[name = tensor("cast_147")]; + tensor fill_mask_29 = reshape(shape = var_1209, x = var_1208)[name = tensor("fill_mask_29")]; tensor x_expanded_29_reps_0 = const()[name = tensor("x_expanded_29_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_29 = tile(reps = x_expanded_29_reps_0, x = var_1790)[name = tensor("x_expanded_29")]; - tensor var_1804 = sub(x = var_17, y = fill_mask_29)[name = tensor("op_1804")]; - tensor var_1805 = mul(x = input_241, y = var_1804)[name = tensor("op_1805")]; - tensor var_1806 = mul(x = x_expanded_29, y = fill_mask_29)[name = tensor("op_1806")]; - tensor filled_window_29 = add(x = var_1805, y = var_1806)[name = tensor("filled_window_29")]; - tensor var_1808 = greater_equal(x = clip_27, y = var_16)[name = tensor("op_1808")]; - tensor var_1809_dtype_0 = const()[name = tensor("op_1809_dtype_0"), val = tensor("fp32")]; - tensor var_1810 = const()[name = tensor("op_1810"), val = tensor([1, 1, 1])]; - tensor var_1809 = cast(dtype = var_1809_dtype_0, x = var_1808)[name = tensor("cast_146")]; - tensor use_roll_29 = reshape(shape = var_1810, x = var_1809)[name = tensor("use_roll_29")]; - tensor var_1812 = sub(x = var_17, y = use_roll_29)[name = tensor("op_1812")]; - tensor var_1813 = mul(x = filled_window_29, y = var_1812)[name = tensor("op_1813")]; - tensor var_1814 = mul(x = rolled_29, y = use_roll_29)[name = tensor("op_1814")]; - tensor input_251 = add(x = var_1813, y = var_1814)[name = tensor("input_251")]; - tensor var_1816 = const()[name = tensor("op_1816"), val = tensor(0x1p+0)]; - tensor var_1817 = add(x = clip_27, y = var_1816)[name = tensor("op_1817")]; - tensor clip_29 = clip(alpha = var_21, beta = var_16, x = var_1817)[name = tensor("clip_29")]; - tensor x_45_axes_0 = const()[name = tensor("x_45_axes_0"), val = tensor([-1])]; - tensor x_45 = layer_norm(axes = x_45_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_251)[name = tensor("x_45")]; - tensor input_253_perm_0 = const()[name = tensor("input_253_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_99_pad_type_0 = const()[name = tensor("inputs_99_pad_type_0"), val = tensor("valid")]; - tensor inputs_99_strides_0 = const()[name = tensor("inputs_99_strides_0"), val = tensor([1])]; - tensor inputs_99_pad_0 = const()[name = tensor("inputs_99_pad_0"), val = tensor([0, 0])]; - tensor inputs_99_dilations_0 = const()[name = tensor("inputs_99_dilations_0"), val = tensor([1])]; - tensor inputs_99_groups_0 = const()[name = tensor("inputs_99_groups_0"), val = tensor(1)]; - tensor input_253 = transpose(perm = input_253_perm_0, x = x_45)[name = tensor("transpose_34")]; - tensor inputs_99 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_99_dilations_0, groups = inputs_99_groups_0, pad = inputs_99_pad_0, pad_type = inputs_99_pad_type_0, strides = inputs_99_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_253)[name = tensor("inputs_99")]; - tensor var_1838_split_sizes_0 = const()[name = tensor("op_1838_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1838_axis_0 = const()[name = tensor("op_1838_axis_0"), val = tensor(1)]; - tensor var_1838_0, tensor var_1838_1 = split(axis = var_1838_axis_0, split_sizes = var_1838_split_sizes_0, x = inputs_99)[name = tensor("op_1838")]; - tensor var_1840 = sigmoid(x = var_1838_1)[name = tensor("op_1840")]; - tensor inputs_101 = mul(x = var_1838_0, y = var_1840)[name = tensor("inputs_101")]; - tensor outputs_aug_29_pad_type_0 = const()[name = tensor("outputs_aug_29_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_29_pad_0 = const()[name = tensor("outputs_aug_29_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_29_groups_0 = const()[name = tensor("outputs_aug_29_groups_0"), val = tensor(256)]; - tensor outputs_aug_29_strides_0 = const()[name = tensor("outputs_aug_29_strides_0"), val = tensor([1])]; - tensor outputs_aug_29_dilations_0 = const()[name = tensor("outputs_aug_29_dilations_0"), val = tensor([1])]; - tensor outputs_aug_29 = conv(dilations = outputs_aug_29_dilations_0, groups = outputs_aug_29_groups_0, pad = outputs_aug_29_pad_0, pad_type = outputs_aug_29_pad_type_0, strides = outputs_aug_29_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_101)[name = tensor("outputs_aug_29")]; - tensor input_255_begin_0 = const()[name = tensor("input_255_begin_0"), val = tensor([0, 0, 0])]; - tensor input_255_end_0 = const()[name = tensor("input_255_end_0"), val = tensor([1, 256, 16])]; - tensor input_255_end_mask_0 = const()[name = tensor("input_255_end_mask_0"), val = tensor([true, true, false])]; - tensor input_255 = slice_by_index(begin = input_255_begin_0, end = input_255_end_0, end_mask = input_255_end_mask_0, x = outputs_aug_29)[name = tensor("input_255")]; - tensor inputs_103 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_255)[name = tensor("inputs_103")]; - tensor input_257 = silu(x = inputs_103)[name = tensor("input_257")]; - tensor input_259_pad_type_0 = const()[name = tensor("input_259_pad_type_0"), val = tensor("valid")]; - tensor input_259_strides_0 = const()[name = tensor("input_259_strides_0"), val = tensor([1])]; - tensor input_259_pad_0 = const()[name = tensor("input_259_pad_0"), val = tensor([0, 0])]; - tensor input_259_dilations_0 = const()[name = tensor("input_259_dilations_0"), val = tensor([1])]; - tensor input_259_groups_0 = const()[name = tensor("input_259_groups_0"), val = tensor(1)]; - tensor input_259 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_259_dilations_0, groups = input_259_groups_0, pad = input_259_pad_0, pad_type = input_259_pad_type_0, strides = input_259_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_257)[name = tensor("input_259")]; - tensor conv_out_29_perm_0 = const()[name = tensor("conv_out_29_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1871_begin_0 = const()[name = tensor("op_1871_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1871_end_0 = const()[name = tensor("op_1871_end_0"), val = tensor([1, 16, 256])]; - tensor var_1871_end_mask_0 = const()[name = tensor("op_1871_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_29 = transpose(perm = conv_out_29_perm_0, x = input_259)[name = tensor("transpose_33")]; - tensor var_1871 = slice_by_index(begin = var_1871_begin_0, end = var_1871_end_0, end_mask = var_1871_end_mask_0, x = conv_out_29)[name = tensor("op_1871")]; - tensor var_1874_begin_0 = const()[name = tensor("op_1874_begin_0"), val = tensor([0, 3, 0])]; - tensor var_1874_end_0 = const()[name = tensor("op_1874_end_0"), val = tensor([1, 1, 256])]; - tensor var_1874_end_mask_0 = const()[name = tensor("op_1874_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1874 = slice_by_index(begin = var_1874_begin_0, end = var_1874_end_0, end_mask = var_1874_end_mask_0, x = x_39)[name = tensor("op_1874")]; - tensor var_1877_begin_0 = const()[name = tensor("op_1877_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1877_end_0 = const()[name = tensor("op_1877_end_0"), val = tensor([1, 16, 256])]; - tensor var_1877_end_mask_0 = const()[name = tensor("op_1877_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1877 = slice_by_index(begin = var_1877_begin_0, end = var_1877_end_0, end_mask = var_1877_end_mask_0, x = input_251)[name = tensor("op_1877")]; + tensor x_expanded_29 = tile(reps = x_expanded_29_reps_0, x = var_1199)[name = tensor("x_expanded_29")]; + tensor var_1213 = sub(x = var_17, y = fill_mask_29)[name = tensor("op_1213")]; + tensor var_1214 = mul(x = window_35, y = var_1213)[name = tensor("op_1214")]; + tensor var_1215 = mul(x = x_expanded_29, y = fill_mask_29)[name = tensor("op_1215")]; + tensor filled_window_29 = add(x = var_1214, y = var_1215)[name = tensor("filled_window_29")]; + tensor var_1217 = greater_equal(x = clip_27, y = var_16)[name = tensor("op_1217")]; + tensor var_1218_dtype_0 = const()[name = tensor("op_1218_dtype_0"), val = tensor("fp32")]; + tensor var_1219 = const()[name = tensor("op_1219"), val = tensor([1, 1, 1])]; + tensor var_1218 = cast(dtype = var_1218_dtype_0, x = var_1217)[name = tensor("cast_146")]; + tensor use_roll_29 = reshape(shape = var_1219, x = var_1218)[name = tensor("use_roll_29")]; + tensor var_1221 = sub(x = var_17, y = use_roll_29)[name = tensor("op_1221")]; + tensor var_1222 = mul(x = filled_window_29, y = var_1221)[name = tensor("op_1222")]; + tensor var_1223 = mul(x = rolled_29, y = use_roll_29)[name = tensor("op_1223")]; + tensor window_37 = add(x = var_1222, y = var_1223)[name = tensor("window_37")]; + tensor var_1225 = const()[name = tensor("op_1225"), val = tensor(0x1p+0)]; + tensor var_1226 = add(x = clip_27, y = var_1225)[name = tensor("op_1226")]; + tensor clip_29 = clip(alpha = var_21, beta = var_16, x = var_1226)[name = tensor("clip_29")]; + tensor var_1229_begin_0 = const()[name = tensor("op_1229_begin_0"), val = tensor([0, 3, 0])]; + tensor var_1229_end_0 = const()[name = tensor("op_1229_end_0"), val = tensor([1, 1, 256])]; + tensor var_1229_end_mask_0 = const()[name = tensor("op_1229_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1229 = slice_by_index(begin = var_1229_begin_0, end = var_1229_end_0, end_mask = var_1229_end_mask_0, x = x_21)[name = tensor("op_1229")]; + tensor var_1232_begin_0 = const()[name = tensor("op_1232_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1232_end_0 = const()[name = tensor("op_1232_end_0"), val = tensor([1, 16, 256])]; + tensor var_1232_end_mask_0 = const()[name = tensor("op_1232_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1232 = slice_by_index(begin = var_1232_begin_0, end = var_1232_end_0, end_mask = var_1232_end_mask_0, x = window_37)[name = tensor("op_1232")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1877, var_1874))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1232, var_1229))[name = tensor("rolled")]; tensor clip_30 = clip(alpha = var_21, beta = var_20, x = clip_29)[name = tensor("clip_30")]; - tensor var_1882 = equal(x = encoder__conv_positions, y = clip_30)[name = tensor("op_1882")]; - tensor var_1883_dtype_0 = const()[name = tensor("op_1883_dtype_0"), val = tensor("fp32")]; - tensor var_1884 = const()[name = tensor("op_1884"), val = tensor([1, 16, 1])]; - tensor var_1883 = cast(dtype = var_1883_dtype_0, x = var_1882)[name = tensor("cast_145")]; - tensor fill_mask = reshape(shape = var_1884, x = var_1883)[name = tensor("fill_mask")]; + tensor var_1237 = equal(x = encoder__conv_positions, y = clip_30)[name = tensor("op_1237")]; + tensor var_1238_dtype_0 = const()[name = tensor("op_1238_dtype_0"), val = tensor("fp32")]; + tensor var_1239 = const()[name = tensor("op_1239"), val = tensor([1, 16, 1])]; + tensor var_1238 = cast(dtype = var_1238_dtype_0, x = var_1237)[name = tensor("cast_145")]; + tensor fill_mask = reshape(shape = var_1239, x = var_1238)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1874)[name = tensor("x_expanded")]; - tensor var_1888 = sub(x = var_17, y = fill_mask)[name = tensor("op_1888")]; - tensor var_1889 = mul(x = input_251, y = var_1888)[name = tensor("op_1889")]; - tensor var_1890 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1890")]; - tensor filled_window = add(x = var_1889, y = var_1890)[name = tensor("filled_window")]; - tensor var_1892 = greater_equal(x = clip_29, y = var_16)[name = tensor("op_1892")]; - tensor var_1893_dtype_0 = const()[name = tensor("op_1893_dtype_0"), val = tensor("fp32")]; - tensor var_1894 = const()[name = tensor("op_1894"), val = tensor([1, 1, 1])]; - tensor var_1893 = cast(dtype = var_1893_dtype_0, x = var_1892)[name = tensor("cast_144")]; - tensor use_roll = reshape(shape = var_1894, x = var_1893)[name = tensor("use_roll")]; - tensor var_1896 = sub(x = var_17, y = use_roll)[name = tensor("op_1896")]; - tensor var_1897 = mul(x = filled_window, y = var_1896)[name = tensor("op_1897")]; - tensor var_1898 = mul(x = rolled, y = use_roll)[name = tensor("op_1898")]; - tensor input_261 = add(x = var_1897, y = var_1898)[name = tensor("input_261")]; - tensor var_1900 = const()[name = tensor("op_1900"), val = tensor(0x1p+0)]; - tensor var_1901 = add(x = clip_29, y = var_1900)[name = tensor("op_1901")]; - tensor clip_31 = clip(alpha = var_21, beta = var_16, x = var_1901)[name = tensor("clip_31")]; - tensor x_47_axes_0 = const()[name = tensor("x_47_axes_0"), val = tensor([-1])]; - tensor x_47 = layer_norm(axes = x_47_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_261)[name = tensor("x_47")]; - tensor input_263_perm_0 = const()[name = tensor("input_263_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_105_pad_type_0 = const()[name = tensor("inputs_105_pad_type_0"), val = tensor("valid")]; - tensor inputs_105_strides_0 = const()[name = tensor("inputs_105_strides_0"), val = tensor([1])]; - tensor inputs_105_pad_0 = const()[name = tensor("inputs_105_pad_0"), val = tensor([0, 0])]; - tensor inputs_105_dilations_0 = const()[name = tensor("inputs_105_dilations_0"), val = tensor([1])]; - tensor inputs_105_groups_0 = const()[name = tensor("inputs_105_groups_0"), val = tensor(1)]; - tensor input_263 = transpose(perm = input_263_perm_0, x = x_47)[name = tensor("transpose_32")]; - tensor inputs_105 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_105_dilations_0, groups = inputs_105_groups_0, pad = inputs_105_pad_0, pad_type = inputs_105_pad_type_0, strides = inputs_105_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_263)[name = tensor("inputs_105")]; - tensor var_1922_split_sizes_0 = const()[name = tensor("op_1922_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1922_axis_0 = const()[name = tensor("op_1922_axis_0"), val = tensor(1)]; - tensor var_1922_0, tensor var_1922_1 = split(axis = var_1922_axis_0, split_sizes = var_1922_split_sizes_0, x = inputs_105)[name = tensor("op_1922")]; - tensor var_1924 = sigmoid(x = var_1922_1)[name = tensor("op_1924")]; - tensor inputs_107 = mul(x = var_1922_0, y = var_1924)[name = tensor("inputs_107")]; + tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1229)[name = tensor("x_expanded")]; + tensor var_1243 = sub(x = var_17, y = fill_mask)[name = tensor("op_1243")]; + tensor var_1244 = mul(x = window_37, y = var_1243)[name = tensor("op_1244")]; + tensor var_1245 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1245")]; + tensor filled_window = add(x = var_1244, y = var_1245)[name = tensor("filled_window")]; + tensor var_1247 = greater_equal(x = clip_29, y = var_16)[name = tensor("op_1247")]; + tensor var_1248_dtype_0 = const()[name = tensor("op_1248_dtype_0"), val = tensor("fp32")]; + tensor var_1249 = const()[name = tensor("op_1249"), val = tensor([1, 1, 1])]; + tensor var_1248 = cast(dtype = var_1248_dtype_0, x = var_1247)[name = tensor("cast_144")]; + tensor use_roll = reshape(shape = var_1249, x = var_1248)[name = tensor("use_roll")]; + tensor var_1251 = sub(x = var_17, y = use_roll)[name = tensor("op_1251")]; + tensor var_1252 = mul(x = filled_window, y = var_1251)[name = tensor("op_1252")]; + tensor var_1253 = mul(x = rolled, y = use_roll)[name = tensor("op_1253")]; + tensor window = add(x = var_1252, y = var_1253)[name = tensor("window")]; + tensor var_1255 = const()[name = tensor("op_1255"), val = tensor(0x1p+0)]; + tensor var_1256 = add(x = clip_29, y = var_1255)[name = tensor("op_1256")]; + tensor clip_31 = clip(alpha = var_21, beta = var_16, x = var_1256)[name = tensor("clip_31")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_30, interleave = input_141_interleave_0, values = (window_33, window_35, window_37, window))[name = tensor("input_141")]; + tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; + tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; + tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_33_pad_type_0 = const()[name = tensor("inputs_33_pad_type_0"), val = tensor("valid")]; + tensor inputs_33_strides_0 = const()[name = tensor("inputs_33_strides_0"), val = tensor([1])]; + tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; + tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; + tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; + tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; + tensor var_1279_split_sizes_0 = const()[name = tensor("op_1279_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1279_axis_0 = const()[name = tensor("op_1279_axis_0"), val = tensor(1)]; + tensor var_1279_0, tensor var_1279_1 = split(axis = var_1279_axis_0, split_sizes = var_1279_split_sizes_0, x = inputs_33)[name = tensor("op_1279")]; + tensor var_1281 = sigmoid(x = var_1279_1)[name = tensor("op_1281")]; + tensor inputs_35 = mul(x = var_1279_0, y = var_1281)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; tensor outputs_aug_strides_0 = const()[name = tensor("outputs_aug_strides_0"), val = tensor([1])]; tensor outputs_aug_dilations_0 = const()[name = tensor("outputs_aug_dilations_0"), val = tensor([1])]; - tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_107)[name = tensor("outputs_aug")]; - tensor input_265_begin_0 = const()[name = tensor("input_265_begin_0"), val = tensor([0, 0, 0])]; - tensor input_265_end_0 = const()[name = tensor("input_265_end_0"), val = tensor([1, 256, 16])]; - tensor input_265_end_mask_0 = const()[name = tensor("input_265_end_mask_0"), val = tensor([true, true, false])]; - tensor input_265 = slice_by_index(begin = input_265_begin_0, end = input_265_end_0, end_mask = input_265_end_mask_0, x = outputs_aug)[name = tensor("input_265")]; - tensor inputs_109 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_265)[name = tensor("inputs_109")]; - tensor input_267 = silu(x = inputs_109)[name = tensor("input_267")]; - tensor input_269_pad_type_0 = const()[name = tensor("input_269_pad_type_0"), val = tensor("valid")]; - tensor input_269_strides_0 = const()[name = tensor("input_269_strides_0"), val = tensor([1])]; - tensor input_269_pad_0 = const()[name = tensor("input_269_pad_0"), val = tensor([0, 0])]; - tensor input_269_dilations_0 = const()[name = tensor("input_269_dilations_0"), val = tensor([1])]; - tensor input_269_groups_0 = const()[name = tensor("input_269_groups_0"), val = tensor(1)]; - tensor input_269 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_269_dilations_0, groups = input_269_groups_0, pad = input_269_pad_0, pad_type = input_269_pad_type_0, strides = input_269_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_267)[name = tensor("input_269")]; - tensor conv_out_31_perm_0 = const()[name = tensor("conv_out_31_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1955_begin_0 = const()[name = tensor("op_1955_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1955_end_0 = const()[name = tensor("op_1955_end_0"), val = tensor([1, 16, 256])]; - tensor var_1955_end_mask_0 = const()[name = tensor("op_1955_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_31 = transpose(perm = conv_out_31_perm_0, x = input_269)[name = tensor("transpose_31")]; - tensor var_1955 = slice_by_index(begin = var_1955_begin_0, end = var_1955_end_0, end_mask = var_1955_end_mask_0, x = conv_out_31)[name = tensor("op_1955")]; - tensor var_1958_interleave_0 = const()[name = tensor("op_1958_interleave_0"), val = tensor(false)]; - tensor var_1958 = concat(axis = var_33, interleave = var_1958_interleave_0, values = (var_1703, var_1787, var_1871, var_1955))[name = tensor("op_1958")]; - tensor input_271 = add(x = x_39, y = var_1958)[name = tensor("input_271")]; - tensor input_273_axes_0 = const()[name = tensor("input_273_axes_0"), val = tensor([-1])]; - tensor input_273 = layer_norm(axes = input_273_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_271)[name = tensor("input_273")]; - tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_273)[name = tensor("linear_35")]; - tensor input_275 = silu(x = inputs)[name = tensor("input_275")]; - tensor input_279 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_275)[name = tensor("linear_36")]; - tensor var_1981 = const()[name = tensor("op_1981"), val = tensor(0x1p-1)]; - tensor var_1982 = mul(x = input_279, y = var_1981)[name = tensor("op_1982")]; - tensor input_281 = add(x = var_1982, y = input_271)[name = tensor("input_281")]; - tensor x_49_axes_0 = const()[name = tensor("x_49_axes_0"), val = tensor([-1])]; - tensor x_49 = layer_norm(axes = x_49_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_281)[name = tensor("x_49")]; + tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_35)[name = tensor("outputs_aug")]; + tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; + tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([4, 256, 16])]; + tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; + tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug)[name = tensor("input_145")]; + tensor inputs_37 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_145)[name = tensor("inputs_37")]; + tensor input_147 = silu(x = inputs_37)[name = tensor("input_147")]; + tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; + tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; + tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; + tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; + tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; + tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; + tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1312_begin_0 = const()[name = tensor("op_1312_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1312_end_0 = const()[name = tensor("op_1312_end_0"), val = tensor([4, 16, 256])]; + tensor var_1312_end_mask_0 = const()[name = tensor("op_1312_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_1312 = slice_by_index(begin = var_1312_begin_0, end = var_1312_end_0, end_mask = var_1312_end_mask_0, x = conv_out_7)[name = tensor("op_1312")]; + tensor var_1314_perm_0 = const()[name = tensor("op_1314_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1314 = transpose(perm = var_1314_perm_0, x = var_1312)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_1314)[name = tensor("input_151")]; + tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; + tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; + tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; + tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; + tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; + tensor var_1337 = const()[name = tensor("op_1337"), val = tensor(0x1p-1)]; + tensor var_1338 = mul(x = input_159, y = var_1337)[name = tensor("op_1338")]; + tensor input_161 = add(x = var_1338, y = input_151)[name = tensor("input_161")]; + tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; + tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; tensor cat_interleave_0 = const()[name = tensor("cat_interleave_0"), val = tensor(false)]; - tensor x_ct = transpose(perm = x_ct_perm_0, x = x_49)[name = tensor("transpose_30")]; + tensor x_ct = transpose(perm = x_ct_perm_0, x = x_25)[name = tensor("transpose_30")]; tensor cat = concat(axis = var_27, interleave = cat_interleave_0, values = (cnn_window, x_ct))[name = tensor("cat")]; tensor conv_out_pad_type_0 = const()[name = tensor("conv_out_pad_type_0"), val = tensor("valid")]; tensor conv_out_strides_0 = const()[name = tensor("conv_out_strides_0"), val = tensor([1])]; @@ -1805,44 +1346,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_2000_begin_0 = const()[name = tensor("op_2000_begin_0"), val = tensor([0, 0, 4])]; - tensor var_2000_end_0 = const()[name = tensor("op_2000_end_0"), val = tensor([1, 256, 22])]; - tensor var_2000_end_mask_0 = const()[name = tensor("op_2000_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_2000_begin_0, end = var_2000_end_0, end_mask = var_2000_end_mask_0, x = cat)[name = tensor("op_2000")]; - tensor input_283_perm_0 = const()[name = tensor("input_283_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2002 = const()[name = tensor("op_2002"), val = tensor([-1])]; - tensor input_283 = transpose(perm = input_283_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_2003 = reduce_l2_norm(axes = var_2002, keep_dims = var_36, x = input_283)[name = tensor("op_2003")]; - tensor const_24 = const()[name = tensor("const_24"), val = tensor(0x1.fffffep+127)]; - tensor clip_32 = clip(alpha = var_12, beta = const_24, x = var_2003)[name = tensor("clip_32")]; - tensor emb = real_div(x = input_283, y = clip_32)[name = tensor("emb")]; - tensor var_2007_axis_0 = const()[name = tensor("op_2007_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_2007_axis_0, values = (var_214, var_678, var_1142, nkv_1))[name = tensor("op_2007")]; - tensor var_2009_axis_0 = const()[name = tensor("op_2009_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_2009_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_2009")]; - tensor var_2011_axis_0 = const()[name = tensor("op_2011_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_2011_axis_0, values = (input_51, input_121, input_191, input_261))[name = tensor("op_2011")]; - tensor var_2013_axis_0 = const()[name = tensor("op_2013_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_2013_axis_0, values = (clip_7, clip_15, clip_23, clip_31))[name = tensor("op_2013")]; - tensor var_2023 = const()[name = tensor("op_2023"), val = tensor(0x1.5798eep-27)]; - tensor var_2029 = const()[name = tensor("op_2029"), val = tensor(0x1.4f8b58p-17)]; - tensor var_2031 = const()[name = tensor("op_2031"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_2032 = const()[name = tensor("op_2032"), val = tensor(true)]; - tensor var_2034 = const()[name = tensor("op_2034"), val = tensor(0x1p+0)]; - tensor var_2038 = const()[name = tensor("op_2038"), val = tensor(-1)]; - tensor var_2043 = const()[name = tensor("op_2043"), val = tensor(0)]; + tensor var_1356_begin_0 = const()[name = tensor("op_1356_begin_0"), val = tensor([0, 0, 4])]; + tensor var_1356_end_0 = const()[name = tensor("op_1356_end_0"), val = tensor([1, 256, 22])]; + tensor var_1356_end_mask_0 = const()[name = tensor("op_1356_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_1356_begin_0, end = var_1356_end_0, end_mask = var_1356_end_mask_0, x = cat)[name = tensor("op_1356")]; + tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1358 = const()[name = tensor("op_1358"), val = tensor([-1])]; + tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; + tensor var_1359 = reduce_l2_norm(axes = var_1358, keep_dims = var_36, x = input_163)[name = tensor("op_1359")]; + tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; + tensor clip_32 = clip(alpha = var_12, beta = const_12, x = var_1359)[name = tensor("clip_32")]; + tensor emb = real_div(x = input_163, y = clip_32)[name = tensor("emb")]; + tensor var_1363_axis_0 = const()[name = tensor("op_1363_axis_0"), val = tensor(0)]; + tensor enc_kv_new = stack(axis = var_1363_axis_0, values = (var_214, var_517, var_820, nkv_1))[name = tensor("op_1363")]; + tensor var_1365_axis_0 = const()[name = tensor("op_1365_axis_0"), val = tensor(0)]; + tensor enc_scale_new = stack(axis = var_1365_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1365")]; + tensor var_1367_axis_0 = const()[name = tensor("op_1367_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1367_axis_0, values = (window_9, window_19, window_29, window))[name = tensor("op_1367")]; + tensor var_1369_axis_0 = const()[name = tensor("op_1369_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1369_axis_0, values = (clip_7, clip_15, clip_23, clip_31))[name = tensor("op_1369")]; + tensor var_1379 = const()[name = tensor("op_1379"), val = tensor(0x1.5798eep-27)]; + tensor var_1384 = const()[name = tensor("op_1384"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1386 = const()[name = tensor("op_1386"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1387 = const()[name = tensor("op_1387"), val = tensor(true)]; + tensor var_1389 = const()[name = tensor("op_1389"), val = tensor(0x1p+0)]; + tensor var_1393 = const()[name = tensor("op_1393"), val = tensor(-1)]; + tensor var_1399 = const()[name = tensor("op_1399"), val = tensor(0)]; tensor pos = const()[name = tensor("pos"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395712)))]; - tensor var_2105_axes_0 = const()[name = tensor("op_2105_axes_0"), val = tensor([2])]; - tensor var_2105 = expand_dims(axes = var_2105_axes_0, x = emb)[name = tensor("op_2105")]; - tensor var_2106 = const()[name = tensor("op_2106"), val = tensor([1, 1, 12, 1])]; - tensor emb_exp = tile(reps = var_2106, x = var_2105)[name = tensor("emb_exp")]; - tensor input_285_interleave_0 = const()[name = tensor("input_285_interleave_0"), val = tensor(false)]; - tensor input_285 = concat(axis = var_2038, interleave = input_285_interleave_0, values = (emb_exp, pos))[name = tensor("input_285")]; - tensor x_51 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_285)[name = tensor("linear_37")]; - tensor var_2113_perm_0 = const()[name = tensor("op_2113_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2117 = const()[name = tensor("op_2117"), val = tensor([12, 4, 256])]; - tensor var_2113 = transpose(perm = var_2113_perm_0, x = x_51)[name = tensor("transpose_28")]; - tensor x_53 = reshape(shape = var_2117, x = var_2113)[name = tensor("x_53")]; + tensor var_1461_axes_0 = const()[name = tensor("op_1461_axes_0"), val = tensor([2])]; + tensor var_1461 = expand_dims(axes = var_1461_axes_0, x = emb)[name = tensor("op_1461")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 12, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1461)[name = tensor("emb_exp")]; + tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; + tensor input_165 = concat(axis = var_1393, interleave = input_165_interleave_0, values = (emb_exp, pos))[name = tensor("input_165")]; + tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; + tensor var_1469_perm_0 = const()[name = tensor("op_1469_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1473 = const()[name = tensor("op_1473"), val = tensor([12, 4, 256])]; + tensor var_1469 = transpose(perm = var_1469_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1473, x = var_1469)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 12, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1853,132 +1394,132 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_2125 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_53)[name = tensor("linear_38")]; - tensor var_2126 = const()[name = tensor("op_2126"), val = tensor([12, 4, 4, 64])]; - tensor var_2127 = reshape(shape = var_2126, x = var_2125)[name = tensor("op_2127")]; + tensor var_1481 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1482 = const()[name = tensor("op_1482"), val = tensor([12, 4, 4, 64])]; + tensor var_1483 = reshape(shape = var_1482, x = var_1481)[name = tensor("op_1483")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2131 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_53)[name = tensor("linear_39")]; - tensor var_2132 = const()[name = tensor("op_2132"), val = tensor(0x1p-3)]; - tensor var_2133 = mul(x = var_2131, y = var_2132)[name = tensor("op_2133")]; - tensor var_2134 = const()[name = tensor("op_2134"), val = tensor([12, 4, 4, 64])]; - tensor var_2135 = reshape(shape = var_2134, x = var_2133)[name = tensor("op_2135")]; + tensor var_1487 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1488 = const()[name = tensor("op_1488"), val = tensor(0x1p-3)]; + tensor var_1489 = mul(x = var_1487, y = var_1488)[name = tensor("op_1489")]; + tensor var_1490 = const()[name = tensor("op_1490"), val = tensor([12, 4, 4, 64])]; + tensor var_1491 = reshape(shape = var_1490, x = var_1489)[name = tensor("op_1491")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2139 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_53)[name = tensor("linear_40")]; - tensor var_2140 = const()[name = tensor("op_2140"), val = tensor([12, 4, 4, 64])]; - tensor var_2141 = reshape(shape = var_2140, x = var_2139)[name = tensor("op_2141")]; + tensor var_1495 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1496 = const()[name = tensor("op_1496"), val = tensor([12, 4, 4, 64])]; + tensor var_1497 = reshape(shape = var_1496, x = var_1495)[name = tensor("op_1497")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_289 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_53)[name = tensor("linear_41")]; + tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_2043, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1399, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; - tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; - tensor clip_33 = clip(alpha = var_2034, beta = const_32, x = s_t_9)[name = tensor("clip_33")]; + tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; + tensor clip_33 = clip(alpha = var_1389, beta = const_20, x = s_t_9)[name = tensor("clip_33")]; tensor sqrt_s_t_9 = sqrt(x = clip_33)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_2135)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_2127)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1491)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1483)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_2153 = const()[name = tensor("op_2153"), val = tensor([1, 4])]; - tensor var_2154 = reshape(shape = var_2153, x = valid_mask)[name = tensor("op_2154")]; - tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_2154)[name = tensor("causal_with_valid_1")]; - tensor var_2156 = const()[name = tensor("op_2156"), val = tensor([4, 1])]; - tensor var_2157 = reshape(shape = var_2156, x = sqrt_s_t_9)[name = tensor("op_2157")]; - tensor M_9 = real_div(x = causal_with_valid_1, y = var_2157)[name = tensor("M_9")]; - tensor var_2159 = mul(x = qk_9, y = M_9)[name = tensor("op_2159")]; + tensor var_1509 = const()[name = tensor("op_1509"), val = tensor([1, 4])]; + tensor var_1510 = reshape(shape = var_1509, x = valid_mask)[name = tensor("op_1510")]; + tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1510)[name = tensor("causal_with_valid_1")]; + tensor var_1512 = const()[name = tensor("op_1512"), val = tensor([4, 1])]; + tensor var_1513 = reshape(shape = var_1512, x = sqrt_s_t_9)[name = tensor("op_1513")]; + tensor M_9 = real_div(x = causal_with_valid_1, y = var_1513)[name = tensor("M_9")]; + tensor var_1515 = mul(x = qk_9, y = M_9)[name = tensor("op_1515")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_2141)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_2159, y = v_9)[name = tensor("inner_9")]; - tensor var_2161_transpose_x_0 = const()[name = tensor("op_2161_transpose_x_0"), val = tensor(false)]; - tensor var_2161_transpose_y_0 = const()[name = tensor("op_2161_transpose_y_0"), val = tensor(false)]; - tensor var_2161 = matmul(transpose_x = var_2161_transpose_x_0, transpose_y = var_2161_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_2161")]; - tensor var_2162 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_2162")]; - tensor var_2163 = const()[name = tensor("op_2163"), val = tensor([1, 1, 4, 1])]; - tensor var_2164 = reshape(shape = var_2163, x = var_2162)[name = tensor("op_2164")]; - tensor cross_9 = mul(x = var_2161, y = var_2164)[name = tensor("cross_9")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1497)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1515, y = v_9)[name = tensor("inner_9")]; + tensor var_1517_transpose_x_0 = const()[name = tensor("op_1517_transpose_x_0"), val = tensor(false)]; + tensor var_1517_transpose_y_0 = const()[name = tensor("op_1517_transpose_y_0"), val = tensor(false)]; + tensor var_1517 = matmul(transpose_x = var_1517_transpose_x_0, transpose_y = var_1517_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1517")]; + tensor var_1518 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1518")]; + tensor var_1519 = const()[name = tensor("op_1519"), val = tensor([1, 1, 4, 1])]; + tensor var_1520 = reshape(shape = var_1519, x = var_1518)[name = tensor("op_1520")]; + tensor cross_9 = mul(x = var_1517, y = var_1520)[name = tensor("cross_9")]; tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; - tensor var_2167 = const()[name = tensor("op_2167"), val = tensor([1, 1, 4, 1])]; - tensor var_2168 = reshape(shape = var_2167, x = valid_mask)[name = tensor("op_2168")]; - tensor v_masked_1 = mul(x = v_9, y = var_2168)[name = tensor("v_masked_1")]; - tensor var_2170 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_2170")]; - tensor var_2172_transpose_x_1 = const()[name = tensor("op_2172_transpose_x_1"), val = tensor(true)]; - tensor var_2172_transpose_y_1 = const()[name = tensor("op_2172_transpose_y_1"), val = tensor(false)]; - tensor var_2172 = matmul(transpose_x = var_2172_transpose_x_1, transpose_y = var_2172_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_2172")]; - tensor new_kv_unnorm_9 = add(x = var_2170, y = var_2172)[name = tensor("new_kv_unnorm_9")]; - tensor var_2174_keep_dims_0 = const()[name = tensor("op_2174_keep_dims_0"), val = tensor(false)]; - tensor var_2174 = reduce_sum(keep_dims = var_2174_keep_dims_0, x = valid_mask)[name = tensor("op_2174")]; - tensor var_2175 = const()[name = tensor("op_2175"), val = tensor([1])]; - tensor var_2176 = reshape(shape = var_2175, x = var_2174)[name = tensor("op_2176")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_2176)[name = tensor("new_scale_9")]; - tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; - tensor clip_34 = clip(alpha = var_2034, beta = const_33, x = new_scale_9)[name = tensor("clip_34")]; + tensor var_1523 = const()[name = tensor("op_1523"), val = tensor([1, 1, 4, 1])]; + tensor var_1524 = reshape(shape = var_1523, x = valid_mask)[name = tensor("op_1524")]; + tensor v_masked_1 = mul(x = v_9, y = var_1524)[name = tensor("v_masked_1")]; + tensor var_1526 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1526")]; + tensor var_1528_transpose_x_1 = const()[name = tensor("op_1528_transpose_x_1"), val = tensor(true)]; + tensor var_1528_transpose_y_1 = const()[name = tensor("op_1528_transpose_y_1"), val = tensor(false)]; + tensor var_1528 = matmul(transpose_x = var_1528_transpose_x_1, transpose_y = var_1528_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1528")]; + tensor new_kv_unnorm_9 = add(x = var_1526, y = var_1528)[name = tensor("new_kv_unnorm_9")]; + tensor var_1530_keep_dims_0 = const()[name = tensor("op_1530_keep_dims_0"), val = tensor(false)]; + tensor var_1530 = reduce_sum(keep_dims = var_1530_keep_dims_0, x = valid_mask)[name = tensor("op_1530")]; + tensor var_1531 = const()[name = tensor("op_1531"), val = tensor([1])]; + tensor var_1532 = reshape(shape = var_1531, x = var_1530)[name = tensor("op_1532")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1532)[name = tensor("new_scale_9")]; + tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; + tensor clip_34 = clip(alpha = var_1389, beta = const_21, x = new_scale_9)[name = tensor("clip_34")]; tensor sqrt_new_scale_1 = sqrt(x = clip_34)[name = tensor("sqrt_new_scale_1")]; - tensor var_2180 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_2180")]; - tensor var_2181_perm_0 = const()[name = tensor("op_2181_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1536 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1536")]; + tensor var_1537_perm_0 = const()[name = tensor("op_1537_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_2181 = transpose(perm = var_2181_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_2031, x = var_2181)[name = tensor("out_27")]; - tensor var_2185 = const()[name = tensor("op_2185"), val = tensor([12, 4, 256])]; - tensor out_29 = reshape(shape = var_2185, x = out_27)[name = tensor("out_29")]; - tensor var_2187 = silu(x = input_289)[name = tensor("op_2187")]; - tensor input_291 = mul(x = var_2187, y = out_29)[name = tensor("input_291")]; - tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_291)[name = tensor("linear_42")]; - tensor input_293 = add(x = x_53, y = ret_out_9)[name = tensor("input_293")]; + tensor var_1537 = transpose(perm = var_1537_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1386, x = var_1537)[name = tensor("out_27")]; + tensor var_1541 = const()[name = tensor("op_1541"), val = tensor([12, 4, 256])]; + tensor out_29 = reshape(shape = var_1541, x = out_27)[name = tensor("out_29")]; + tensor var_1543 = silu(x = input_169)[name = tensor("op_1543")]; + tensor input_171 = mul(x = var_1543, y = out_29)[name = tensor("input_171")]; + tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; + tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_2029, gamma = decoder_norm11_0_weight, x = input_293)[name = tensor("xt_1")]; - tensor var_2197 = const()[name = tensor("op_2197"), val = tensor([1, 12, 4, 256])]; - tensor var_2198 = reshape(shape = var_2197, x = xt_1)[name = tensor("op_2198")]; - tensor var_2199_perm_0 = const()[name = tensor("op_2199_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2202 = const()[name = tensor("op_2202"), val = tensor([4, 12, 256])]; - tensor var_2199 = transpose(perm = var_2199_perm_0, x = var_2198)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_2202, x = var_2199)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1384, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1553 = const()[name = tensor("op_1553"), val = tensor([1, 12, 4, 256])]; + tensor var_1554 = reshape(shape = var_1553, x = xt_1)[name = tensor("op_1554")]; + tensor var_1555_perm_0 = const()[name = tensor("op_1555_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1558 = const()[name = tensor("op_1558"), val = tensor([4, 12, 256])]; + tensor var_1555 = transpose(perm = var_1555_perm_0, x = var_1554)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1558, x = var_1555)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_2225 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_16 = const()[name = tensor("concat_16"), val = tensor([12, 4, 3, 256])]; - tensor var_2227 = reshape(shape = concat_16, x = var_2225)[name = tensor("op_2227")]; - tensor var_2228_axes_0 = const()[name = tensor("op_2228_axes_0"), val = tensor([0])]; - tensor var_2228 = expand_dims(axes = var_2228_axes_0, x = var_2227)[name = tensor("op_2228")]; - tensor var_2229_perm_0 = const()[name = tensor("op_2229_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2230_axes_0 = const()[name = tensor("op_2230_axes_0"), val = tensor([-2])]; - tensor var_2229 = transpose(perm = var_2229_perm_0, x = var_2228)[name = tensor("transpose_21")]; - tensor var_2230 = squeeze(axes = var_2230_axes_0, x = var_2229)[name = tensor("op_2230")]; + tensor var_1581 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_17 = const()[name = tensor("concat_17"), val = tensor([12, 4, 3, 256])]; + tensor var_1583 = reshape(shape = concat_17, x = var_1581)[name = tensor("op_1583")]; + tensor var_1584_axes_0 = const()[name = tensor("op_1584_axes_0"), val = tensor([0])]; + tensor var_1584 = expand_dims(axes = var_1584_axes_0, x = var_1583)[name = tensor("op_1584")]; + tensor var_1585_perm_0 = const()[name = tensor("op_1585_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1586_axes_0 = const()[name = tensor("op_1586_axes_0"), val = tensor([-2])]; + tensor var_1585 = transpose(perm = var_1585_perm_0, x = var_1584)[name = tensor("transpose_21")]; + tensor var_1586 = squeeze(axes = var_1586_axes_0, x = var_1585)[name = tensor("op_1586")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 12, 4, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_2230)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1586)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 12, 4, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_2230)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1586)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 12, 4, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_2230)[name = tensor("v_11")]; - tensor var_2238 = const()[name = tensor("op_2238"), val = tensor([12, 16, 64])]; - tensor var_2239 = reshape(shape = var_2238, x = q_11)[name = tensor("op_2239")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1586)[name = tensor("v_11")]; + tensor var_1594 = const()[name = tensor("op_1594"), val = tensor([12, 16, 64])]; + tensor var_1595 = reshape(shape = var_1594, x = q_11)[name = tensor("op_1595")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2245 = const()[name = tensor("op_2245"), val = tensor([12, 16, 64])]; - tensor var_2246 = reshape(shape = var_2245, x = k_11)[name = tensor("op_2246")]; + tensor var_1601 = const()[name = tensor("op_1601"), val = tensor([12, 16, 64])]; + tensor var_1602 = reshape(shape = var_1601, x = k_11)[name = tensor("op_1602")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2252 = const()[name = tensor("op_2252"), val = tensor([12, 16, 64])]; - tensor var_2253 = reshape(shape = var_2252, x = v_11)[name = tensor("op_2253")]; + tensor var_1608 = const()[name = tensor("op_1608"), val = tensor([12, 16, 64])]; + tensor var_1609 = reshape(shape = var_1608, x = v_11)[name = tensor("op_1609")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2256 = const()[name = tensor("op_2256"), val = tensor([4, 4, 12, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_2239)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_2256, x = q_13)[name = tensor("q_15")]; - tensor var_2258 = const()[name = tensor("op_2258"), val = tensor([4, 4, 12, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_2246)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_2258, x = k_13)[name = tensor("k_15")]; - tensor var_2260 = const()[name = tensor("op_2260"), val = tensor([4, 4, 12, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_2253)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_2260, x = v_13)[name = tensor("v_15")]; + tensor var_1612 = const()[name = tensor("op_1612"), val = tensor([4, 4, 12, 64])]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1595)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1612, x = q_13)[name = tensor("q_15")]; + tensor var_1614 = const()[name = tensor("op_1614"), val = tensor([4, 4, 12, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1602)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1614, x = k_13)[name = tensor("k_15")]; + tensor var_1616 = const()[name = tensor("op_1616"), val = tensor([4, 4, 12, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1609)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1616, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -1989,30 +1530,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_2263 = const()[name = tensor("op_2263"), val = tensor([2, 0, 1, 3])]; - tensor var_2268 = const()[name = tensor("op_2268"), val = tensor([48, 256])]; - tensor var_2264 = transpose(perm = var_2263, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_2268, x = var_2264)[name = tensor("attn_output_3")]; + tensor var_1619 = const()[name = tensor("op_1619"), val = tensor([2, 0, 1, 3])]; + tensor var_1624 = const()[name = tensor("op_1624"), val = tensor([48, 256])]; + tensor var_1620 = transpose(perm = var_1619, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1624, x = var_1620)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_2272 = const()[name = tensor("op_2272"), val = tensor([12, 4, 256])]; - tensor attn_output_7 = reshape(shape = var_2272, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1628 = const()[name = tensor("op_1628"), val = tensor([12, 4, 256])]; + tensor attn_output_7 = reshape(shape = var_1628, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; - tensor input_295 = add(x = query_1, y = sa2_out_1)[name = tensor("input_295")]; - tensor input_297_axes_0 = const()[name = tensor("input_297_axes_0"), val = tensor([-1])]; - tensor input_297 = layer_norm(axes = input_297_axes_0, beta = decoder_norm21_0_bias, epsilon = var_2029, gamma = decoder_norm21_0_weight, x = input_295)[name = tensor("input_297")]; - tensor input_299 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_297)[name = tensor("linear_45")]; - tensor input_301 = relu(x = input_299)[name = tensor("input_301")]; - tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_301)[name = tensor("linear_46")]; - tensor input_303 = add(x = input_297, y = ffn_out_1)[name = tensor("input_303")]; + tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; + tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1384, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; + tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; + tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; + tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_2029, gamma = decoder_norm22_0_weight, x = input_303)[name = tensor("xt_3")]; - tensor var_2292 = const()[name = tensor("op_2292"), val = tensor([1, 4, 12, 256])]; - tensor x_55 = reshape(shape = var_2292, x = xt_3)[name = tensor("x_55")]; - tensor var_2294_perm_0 = const()[name = tensor("op_2294_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2298 = const()[name = tensor("op_2298"), val = tensor([12, 4, 256])]; - tensor var_2294 = transpose(perm = var_2294_perm_0, x = x_55)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_2298, x = var_2294)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1384, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1648 = const()[name = tensor("op_1648"), val = tensor([1, 4, 12, 256])]; + tensor x_31 = reshape(shape = var_1648, x = xt_3)[name = tensor("x_31")]; + tensor var_1650_perm_0 = const()[name = tensor("op_1650_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1654 = const()[name = tensor("op_1654"), val = tensor([12, 4, 256])]; + tensor var_1650 = transpose(perm = var_1650_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1654, x = var_1650)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 12, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -2023,120 +1564,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_2306 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_2307 = const()[name = tensor("op_2307"), val = tensor([12, 4, 4, 64])]; - tensor var_2308 = reshape(shape = var_2307, x = var_2306)[name = tensor("op_2308")]; + tensor var_1662 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1663 = const()[name = tensor("op_1663"), val = tensor([12, 4, 4, 64])]; + tensor var_1664 = reshape(shape = var_1663, x = var_1662)[name = tensor("op_1664")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2312 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_2313 = const()[name = tensor("op_2313"), val = tensor(0x1p-3)]; - tensor var_2314 = mul(x = var_2312, y = var_2313)[name = tensor("op_2314")]; - tensor var_2315 = const()[name = tensor("op_2315"), val = tensor([12, 4, 4, 64])]; - tensor var_2316 = reshape(shape = var_2315, x = var_2314)[name = tensor("op_2316")]; + tensor var_1668 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1669 = const()[name = tensor("op_1669"), val = tensor(0x1p-3)]; + tensor var_1670 = mul(x = var_1668, y = var_1669)[name = tensor("op_1670")]; + tensor var_1671 = const()[name = tensor("op_1671"), val = tensor([12, 4, 4, 64])]; + tensor var_1672 = reshape(shape = var_1671, x = var_1670)[name = tensor("op_1672")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2320 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_2321 = const()[name = tensor("op_2321"), val = tensor([12, 4, 4, 64])]; - tensor var_2322 = reshape(shape = var_2321, x = var_2320)[name = tensor("op_2322")]; + tensor var_1676 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1677 = const()[name = tensor("op_1677"), val = tensor([12, 4, 4, 64])]; + tensor var_1678 = reshape(shape = var_1677, x = var_1676)[name = tensor("op_1678")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_307 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; + tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; - tensor const_44 = const()[name = tensor("const_44"), val = tensor(0x1.fffffep+127)]; - tensor clip_35 = clip(alpha = var_2034, beta = const_44, x = s_t)[name = tensor("clip_35")]; + tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; + tensor clip_35 = clip(alpha = var_1389, beta = const_32, x = s_t)[name = tensor("clip_35")]; tensor sqrt_s_t = sqrt(x = clip_35)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_2316)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_2308)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1672)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1664)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_2337 = const()[name = tensor("op_2337"), val = tensor([4, 1])]; - tensor var_2338 = reshape(shape = var_2337, x = sqrt_s_t)[name = tensor("op_2338")]; - tensor M = real_div(x = causal_with_valid_1, y = var_2338)[name = tensor("M")]; - tensor var_2340 = mul(x = qk, y = M)[name = tensor("op_2340")]; + tensor var_1693 = const()[name = tensor("op_1693"), val = tensor([4, 1])]; + tensor var_1694 = reshape(shape = var_1693, x = sqrt_s_t)[name = tensor("op_1694")]; + tensor M = real_div(x = causal_with_valid_1, y = var_1694)[name = tensor("M")]; + tensor var_1696 = mul(x = qk, y = M)[name = tensor("op_1696")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_2322)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_2340, y = v_17)[name = tensor("inner")]; - tensor var_2342_transpose_x_0 = const()[name = tensor("op_2342_transpose_x_0"), val = tensor(false)]; - tensor var_2342_transpose_y_0 = const()[name = tensor("op_2342_transpose_y_0"), val = tensor(false)]; - tensor var_2342 = matmul(transpose_x = var_2342_transpose_x_0, transpose_y = var_2342_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_2342")]; - tensor var_2343 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_2343")]; - tensor var_2344 = const()[name = tensor("op_2344"), val = tensor([1, 1, 4, 1])]; - tensor var_2345 = reshape(shape = var_2344, x = var_2343)[name = tensor("op_2345")]; - tensor cross = mul(x = var_2342, y = var_2345)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1678)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1696, y = v_17)[name = tensor("inner")]; + tensor var_1698_transpose_x_0 = const()[name = tensor("op_1698_transpose_x_0"), val = tensor(false)]; + tensor var_1698_transpose_y_0 = const()[name = tensor("op_1698_transpose_y_0"), val = tensor(false)]; + tensor var_1698 = matmul(transpose_x = var_1698_transpose_x_0, transpose_y = var_1698_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1698")]; + tensor var_1699 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1699")]; + tensor var_1700 = const()[name = tensor("op_1700"), val = tensor([1, 1, 4, 1])]; + tensor var_1701 = reshape(shape = var_1700, x = var_1699)[name = tensor("op_1701")]; + tensor cross = mul(x = var_1698, y = var_1701)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_2168)[name = tensor("v_masked")]; - tensor var_2351 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_2351")]; - tensor var_2353_transpose_x_1 = const()[name = tensor("op_2353_transpose_x_1"), val = tensor(true)]; - tensor var_2353_transpose_y_1 = const()[name = tensor("op_2353_transpose_y_1"), val = tensor(false)]; - tensor var_2353 = matmul(transpose_x = var_2353_transpose_x_1, transpose_y = var_2353_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_2353")]; - tensor new_kv_unnorm = add(x = var_2351, y = var_2353)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_2176)[name = tensor("new_scale")]; - tensor const_45 = const()[name = tensor("const_45"), val = tensor(0x1.fffffep+127)]; - tensor clip_36 = clip(alpha = var_2034, beta = const_45, x = new_scale)[name = tensor("clip_36")]; + tensor v_masked = mul(x = v_17, y = var_1524)[name = tensor("v_masked")]; + tensor var_1707 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1707")]; + tensor var_1709_transpose_x_1 = const()[name = tensor("op_1709_transpose_x_1"), val = tensor(true)]; + tensor var_1709_transpose_y_1 = const()[name = tensor("op_1709_transpose_y_1"), val = tensor(false)]; + tensor var_1709 = matmul(transpose_x = var_1709_transpose_x_1, transpose_y = var_1709_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1709")]; + tensor new_kv_unnorm = add(x = var_1707, y = var_1709)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1532)[name = tensor("new_scale")]; + tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; + tensor clip_36 = clip(alpha = var_1389, beta = const_33, x = new_scale)[name = tensor("clip_36")]; tensor sqrt_new_scale = sqrt(x = clip_36)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_2362_perm_0 = const()[name = tensor("op_2362_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1718_perm_0 = const()[name = tensor("op_1718_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_2362 = transpose(perm = var_2362_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_2031, x = var_2362)[name = tensor("out_33")]; - tensor var_2366 = const()[name = tensor("op_2366"), val = tensor([12, 4, 256])]; - tensor out = reshape(shape = var_2366, x = out_33)[name = tensor("out")]; - tensor var_2368 = silu(x = input_307)[name = tensor("op_2368")]; - tensor input_309 = mul(x = var_2368, y = out)[name = tensor("input_309")]; - tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_309)[name = tensor("linear_51")]; - tensor input_311 = add(x = x, y = ret_out)[name = tensor("input_311")]; + tensor var_1718 = transpose(perm = var_1718_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1386, x = var_1718)[name = tensor("out_33")]; + tensor var_1722 = const()[name = tensor("op_1722"), val = tensor([12, 4, 256])]; + tensor out = reshape(shape = var_1722, x = out_33)[name = tensor("out")]; + tensor var_1724 = silu(x = input_187)[name = tensor("op_1724")]; + tensor input_189 = mul(x = var_1724, y = out)[name = tensor("input_189")]; + tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; + tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_2029, gamma = decoder_norm11_1_weight, x = input_311)[name = tensor("xt_5")]; - tensor var_2378 = const()[name = tensor("op_2378"), val = tensor([1, 12, 4, 256])]; - tensor var_2379 = reshape(shape = var_2378, x = xt_5)[name = tensor("op_2379")]; - tensor var_2380_perm_0 = const()[name = tensor("op_2380_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2383 = const()[name = tensor("op_2383"), val = tensor([4, 12, 256])]; - tensor var_2380 = transpose(perm = var_2380_perm_0, x = var_2379)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_2383, x = var_2380)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1384, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1734 = const()[name = tensor("op_1734"), val = tensor([1, 12, 4, 256])]; + tensor var_1735 = reshape(shape = var_1734, x = xt_5)[name = tensor("op_1735")]; + tensor var_1736_perm_0 = const()[name = tensor("op_1736_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1739 = const()[name = tensor("op_1739"), val = tensor([4, 12, 256])]; + tensor var_1736 = transpose(perm = var_1736_perm_0, x = var_1735)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1739, x = var_1736)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_2406 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_17 = const()[name = tensor("concat_17"), val = tensor([12, 4, 3, 256])]; - tensor var_2408 = reshape(shape = concat_17, x = var_2406)[name = tensor("op_2408")]; - tensor var_2409_axes_0 = const()[name = tensor("op_2409_axes_0"), val = tensor([0])]; - tensor var_2409 = expand_dims(axes = var_2409_axes_0, x = var_2408)[name = tensor("op_2409")]; - tensor var_2410_perm_0 = const()[name = tensor("op_2410_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2411_axes_0 = const()[name = tensor("op_2411_axes_0"), val = tensor([-2])]; - tensor var_2410 = transpose(perm = var_2410_perm_0, x = var_2409)[name = tensor("transpose_8")]; - tensor var_2411 = squeeze(axes = var_2411_axes_0, x = var_2410)[name = tensor("op_2411")]; + tensor var_1762 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_18 = const()[name = tensor("concat_18"), val = tensor([12, 4, 3, 256])]; + tensor var_1764 = reshape(shape = concat_18, x = var_1762)[name = tensor("op_1764")]; + tensor var_1765_axes_0 = const()[name = tensor("op_1765_axes_0"), val = tensor([0])]; + tensor var_1765 = expand_dims(axes = var_1765_axes_0, x = var_1764)[name = tensor("op_1765")]; + tensor var_1766_perm_0 = const()[name = tensor("op_1766_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1767_axes_0 = const()[name = tensor("op_1767_axes_0"), val = tensor([-2])]; + tensor var_1766 = transpose(perm = var_1766_perm_0, x = var_1765)[name = tensor("transpose_8")]; + tensor var_1767 = squeeze(axes = var_1767_axes_0, x = var_1766)[name = tensor("op_1767")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 12, 4, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_2411)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1767)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 12, 4, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_2411)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1767)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 12, 4, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_2411)[name = tensor("v_19")]; - tensor var_2419 = const()[name = tensor("op_2419"), val = tensor([12, 16, 64])]; - tensor var_2420 = reshape(shape = var_2419, x = q_19)[name = tensor("op_2420")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1767)[name = tensor("v_19")]; + tensor var_1775 = const()[name = tensor("op_1775"), val = tensor([12, 16, 64])]; + tensor var_1776 = reshape(shape = var_1775, x = q_19)[name = tensor("op_1776")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2426 = const()[name = tensor("op_2426"), val = tensor([12, 16, 64])]; - tensor var_2427 = reshape(shape = var_2426, x = k_19)[name = tensor("op_2427")]; + tensor var_1782 = const()[name = tensor("op_1782"), val = tensor([12, 16, 64])]; + tensor var_1783 = reshape(shape = var_1782, x = k_19)[name = tensor("op_1783")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2433 = const()[name = tensor("op_2433"), val = tensor([12, 16, 64])]; - tensor var_2434 = reshape(shape = var_2433, x = v_19)[name = tensor("op_2434")]; + tensor var_1789 = const()[name = tensor("op_1789"), val = tensor([12, 16, 64])]; + tensor var_1790 = reshape(shape = var_1789, x = v_19)[name = tensor("op_1790")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2437 = const()[name = tensor("op_2437"), val = tensor([4, 4, 12, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_2420)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_2437, x = q_21)[name = tensor("q")]; - tensor var_2439 = const()[name = tensor("op_2439"), val = tensor([4, 4, 12, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_2427)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_2439, x = k_21)[name = tensor("k")]; - tensor var_2441 = const()[name = tensor("op_2441"), val = tensor([4, 4, 12, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_2434)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_2441, x = v_21)[name = tensor("v")]; + tensor var_1793 = const()[name = tensor("op_1793"), val = tensor([4, 4, 12, 64])]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1776)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1793, x = q_21)[name = tensor("q")]; + tensor var_1795 = const()[name = tensor("op_1795"), val = tensor([4, 4, 12, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1783)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1795, x = k_21)[name = tensor("k")]; + tensor var_1797 = const()[name = tensor("op_1797"), val = tensor([4, 4, 12, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1790)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1797, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -2147,50 +1688,50 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_2444 = const()[name = tensor("op_2444"), val = tensor([2, 0, 1, 3])]; - tensor var_2449 = const()[name = tensor("op_2449"), val = tensor([48, 256])]; - tensor var_2445 = transpose(perm = var_2444, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_2449, x = var_2445)[name = tensor("attn_output_11")]; + tensor var_1800 = const()[name = tensor("op_1800"), val = tensor([2, 0, 1, 3])]; + tensor var_1805 = const()[name = tensor("op_1805"), val = tensor([48, 256])]; + tensor var_1801 = transpose(perm = var_1800, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1805, x = var_1801)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_2453 = const()[name = tensor("op_2453"), val = tensor([12, 4, 256])]; - tensor attn_output = reshape(shape = var_2453, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1809 = const()[name = tensor("op_1809"), val = tensor([12, 4, 256])]; + tensor attn_output = reshape(shape = var_1809, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; - tensor input_313 = add(x = query_5, y = sa2_out)[name = tensor("input_313")]; - tensor input_315_axes_0 = const()[name = tensor("input_315_axes_0"), val = tensor([-1])]; - tensor input_315 = layer_norm(axes = input_315_axes_0, beta = decoder_norm21_1_bias, epsilon = var_2029, gamma = decoder_norm21_1_weight, x = input_313)[name = tensor("input_315")]; - tensor input_317 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_315)[name = tensor("linear_54")]; - tensor input_319 = relu(x = input_317)[name = tensor("input_319")]; - tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_319)[name = tensor("linear_55")]; - tensor input_321 = add(x = input_315, y = ffn_out)[name = tensor("input_321")]; + tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; + tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1384, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; + tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; + tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; + tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_2029, gamma = decoder_norm22_1_weight, x = input_321)[name = tensor("xt")]; - tensor var_2473 = const()[name = tensor("op_2473"), val = tensor([1, 4, 12, 256])]; - tensor input = reshape(shape = var_2473, x = xt)[name = tensor("input")]; - tensor var_2475 = const()[name = tensor("op_2475"), val = tensor([-1])]; - tensor var_2476 = reduce_l2_norm(axes = var_2475, keep_dims = var_2032, x = input)[name = tensor("op_2476")]; - tensor const_54 = const()[name = tensor("const_54"), val = tensor(0x1.fffffep+127)]; - tensor clip_37 = clip(alpha = var_2023, beta = const_54, x = var_2476)[name = tensor("clip_37")]; - tensor var_2478 = real_div(x = input, y = clip_37)[name = tensor("op_2478")]; - tensor concat_21 = const()[name = tensor("concat_21"), val = tensor([4, 1, 256])]; - tensor reshape_0 = reshape(shape = concat_21, x = emb)[name = tensor("reshape_0")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1384, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1829 = const()[name = tensor("op_1829"), val = tensor([1, 4, 12, 256])]; + tensor input = reshape(shape = var_1829, x = xt)[name = tensor("input")]; + tensor var_1831 = const()[name = tensor("op_1831"), val = tensor([-1])]; + tensor var_1832 = reduce_l2_norm(axes = var_1831, keep_dims = var_1387, x = input)[name = tensor("op_1832")]; + tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; + tensor clip_37 = clip(alpha = var_1379, beta = const_42, x = var_1832)[name = tensor("clip_37")]; + tensor var_1834 = real_div(x = input, y = clip_37)[name = tensor("op_1834")]; + tensor concat_22 = const()[name = tensor("concat_22"), val = tensor([4, 1, 256])]; + tensor reshape_0 = reshape(shape = concat_22, x = emb)[name = tensor("reshape_0")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_22 = const()[name = tensor("concat_22"), val = tensor([4, 256, 12])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_2478)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_22, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_23 = const()[name = tensor("concat_23"), val = tensor([4, 256, 12])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1834)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_23, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = reshape_0, y = reshape_1)[name = tensor("matmul_2")]; - tensor concat_25 = const()[name = tensor("concat_25"), val = tensor([1, 4, 12])]; - tensor reshape_2 = reshape(shape = concat_25, x = matmul_2)[name = tensor("reshape_2")]; + tensor concat_26 = const()[name = tensor("concat_26"), val = tensor([1, 4, 12])]; + tensor reshape_2 = reshape(shape = concat_26, x = matmul_2)[name = tensor("reshape_2")]; tensor output_begin_0 = const()[name = tensor("output_begin_0"), val = tensor([0, 0, 1])]; tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 4, 11])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = reshape_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_2482")]; - tensor var_2484_axis_0 = const()[name = tensor("op_2484_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_2484_axis_0, values = (var_2180, nkv))[name = tensor("op_2484")]; - tensor var_2486_axis_0 = const()[name = tensor("op_2486_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_2486_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_2486")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1838")]; + tensor var_1840_axis_0 = const()[name = tensor("op_1840_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1840_axis_0, values = (var_1536, nkv))[name = tensor("op_1840")]; + tensor var_1842_axis_0 = const()[name = tensor("op_1842_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1842_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1842")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/DIHARD II/ls_eend_dih2_400ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/DIHARD II/ls_eend_dih2_400ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index c0d2443b8be56e613d56182fc2ffacc520eacd09..71f349eb3c898557b306014edff7e4eeb1c3de11 100644 --- a/optimized/DIHARD II/ls_eend_dih2_400ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/DIHARD II/ls_eend_dih2_400ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:aac5c87ebc1c779bf4c4e4303bb6f23f055ce9aca3ea36673514b810ca265914 -size 311059 +oid sha256:c35c1905acf8a9f72d87e5a2d12a54e03cfc2c001c115385773be6375755ca77 +size 238964 diff --git a/optimized/DIHARD II/ls_eend_dih2_400ms.mlpackage/Manifest.json b/optimized/DIHARD II/ls_eend_dih2_400ms.mlpackage/Manifest.json index 112ce43e25d79100c5e2919cc9426a1a68f3781f..3c8a1b27ee4b3e5d3afc1f324a6627c6b4260e15 100644 --- a/optimized/DIHARD II/ls_eend_dih2_400ms.mlpackage/Manifest.json +++ b/optimized/DIHARD II/ls_eend_dih2_400ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "EB934AFC-D4B9-4B35-89BC-CE16F55E6E7B": { - "author": "com.apple.CoreML", - "description": "CoreML Model Weights", - "name": "weights", - "path": "com.apple.CoreML/weights" - }, - "ED03D0C6-C019-476C-B6E4-C0D77F380ED2": { + "3644A56F-302E-4E0F-BCE6-AD11BED39D3B": { "author": "com.apple.CoreML", "description": "CoreML Model Specification", "name": "model.mlmodel", "path": "com.apple.CoreML/model.mlmodel" + }, + "6C888EF1-8720-490B-8621-4CA201933366": { + "author": "com.apple.CoreML", + "description": "CoreML Model Weights", + "name": "weights", + "path": "com.apple.CoreML/weights" } }, - "rootModelIdentifier": "ED03D0C6-C019-476C-B6E4-C0D77F380ED2" + "rootModelIdentifier": "3644A56F-302E-4E0F-BCE6-AD11BED39D3B" } diff --git a/optimized/DIHARD II/ls_eend_dih2_500ms.mlmodelc/analytics/coremldata.bin b/optimized/DIHARD II/ls_eend_dih2_500ms.mlmodelc/analytics/coremldata.bin index 6da75e652f29d38526d6bcbfc49ed1cf61028a58..4892101f3b8a7105e518f22305a62790c0e8b815 100644 --- a/optimized/DIHARD II/ls_eend_dih2_500ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/DIHARD II/ls_eend_dih2_500ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:656f7fba629d0c0e0ae269f0838578f9075d245f4fd7d312ce818324a24445be +oid sha256:e99ed8b1a266f40584bb85092532d70583405841dbcf61cc8a4700ac504a0ee3 size 243 diff --git a/optimized/DIHARD II/ls_eend_dih2_500ms.mlmodelc/coremldata.bin b/optimized/DIHARD II/ls_eend_dih2_500ms.mlmodelc/coremldata.bin index 7b1d1435462d3a42b362c1e544c626e08ec48f65..5b328872cc2ae64c1734055c6cb271db364365d3 100644 --- a/optimized/DIHARD II/ls_eend_dih2_500ms.mlmodelc/coremldata.bin +++ b/optimized/DIHARD II/ls_eend_dih2_500ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:529c260c5102b86331ac1d11615b88ed1f6e8c4815e907429ebd73b808ab5b5a -size 1266 +oid sha256:5aa2c7dbcca04f4eb479fd73c956399ca8623fd1f0bb4004c43507cb2f4b3df6 +size 1357 diff --git a/optimized/DIHARD II/ls_eend_dih2_500ms.mlmodelc/metadata.json b/optimized/DIHARD II/ls_eend_dih2_500ms.mlmodelc/metadata.json index 034234e25b070dd25f96248b7ca0d4fc3e2ee883..ad8581d14540968c229bd4379b420f10bdede7b7 100644 --- a/optimized/DIHARD II/ls_eend_dih2_500ms.mlmodelc/metadata.json +++ b/optimized/DIHARD II/ls_eend_dih2_500ms.mlmodelc/metadata.json @@ -96,30 +96,30 @@ "Ios17.reshape" : 108, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 85, - "Split" : 20, + "Ios17.transpose" : 57, + "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 106, "Tile" : 21, - "Ios17.sliceByIndex" : 108, - "Ios16.sigmoid" : 21, + "Ios17.sliceByIndex" : 76, + "Ios16.sigmoid" : 5, "Ios16.reduceSum" : 1, "Ios17.squeeze" : 2, - "Ios17.layerNorm" : 49, - "Ios17.batchNorm" : 20, + "Ios17.layerNorm" : 33, + "Ios17.batchNorm" : 4, "Ios17.cast" : 40, "Ios17.sqrt" : 18, "Ios17.sub" : 40, - "Ios17.conv" : 61, + "Ios17.conv" : 13, "Ios17.realDiv" : 20, "Ios17.linear" : 56, "Ios17.concat" : 26, "Ios17.clip" : 46, "Ios17.greaterEqual" : 20, - "Ios16.silu" : 34, + "Ios16.silu" : 18, "Stack" : 6, "Ios16.cumsum" : 1, - "Ios17.mul" : 143 + "Ios17.mul" : 127 }, "computePrecision" : "Mixed (Float32, Int32)", "isUpdatable" : "0", @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"dih2\", \"model_label\": \"DIHARD II\", \"variant\": \"pipeline\", \"T\": 5, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"dih2\", \"model_label\": \"DIHARD II\", \"variant\": \"pipeline\", \"chunk_size\": 5, \"step_duration_ms\": 500, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/DIHARD II/ls_eend_dih2_500ms.mlmodelc/model.mil b/optimized/DIHARD II/ls_eend_dih2_500ms.mlmodelc/model.mil index d4fb1d80c3e733d6b9ac22e812c68c6db0697bc2..b13035ac5720520676a47ddea8a64f614f91249d 100644 --- a/optimized/DIHARD II/ls_eend_dih2_500ms.mlmodelc/model.mil +++ b/optimized/DIHARD II/ls_eend_dih2_500ms.mlmodelc/model.mil @@ -217,6 +217,7 @@ program(1.0) tensor var_22 = const()[name = tensor("op_22"), val = tensor(0x0p+0)]; tensor var_25 = const()[name = tensor("op_25"), val = tensor(0x1.0c6f7ap-20)]; tensor var_28 = const()[name = tensor("op_28"), val = tensor(2)]; + tensor var_31 = const()[name = tensor("op_31"), val = tensor(0)]; tensor var_34 = const()[name = tensor("op_34"), val = tensor(1)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(0x1.4f8b58p-17)]; tensor var_37 = const()[name = tensor("op_37"), val = tensor(true)]; @@ -263,8 +264,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_181)[name = tensor("transpose_85")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_173)[name = tensor("transpose_86")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_181)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_173)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_197 = const()[name = tensor("op_197"), val = tensor([5, 1])]; tensor var_198 = reshape(shape = var_197, x = sqrt_s_t_1)[name = tensor("op_198")]; @@ -272,7 +273,7 @@ program(1.0) tensor var_200 = mul(x = qk_1, y = M_1)[name = tensor("op_200")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_187)[name = tensor("transpose_84")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_187)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_200, y = v_1)[name = tensor("inner_1")]; tensor var_202_transpose_x_0 = const()[name = tensor("op_202_transpose_x_0"), val = tensor(false)]; tensor var_202_transpose_y_0 = const()[name = tensor("op_202_transpose_y_0"), val = tensor(false)]; @@ -293,7 +294,7 @@ program(1.0) tensor var_215 = real_div(x = new_kv_unnorm_1, y = var_214)[name = tensor("op_215")]; tensor var_216_perm_0 = const()[name = tensor("op_216_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_216 = transpose(perm = var_216_perm_0, x = out_1)[name = tensor("transpose_83")]; + tensor var_216 = transpose(perm = var_216_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_25, x = var_216)[name = tensor("out_3")]; tensor var_220 = const()[name = tensor("op_220"), val = tensor([1, 5, 256])]; tensor out_5 = reshape(shape = var_220, x = out_3)[name = tensor("out_5")]; @@ -341,364 +342,210 @@ program(1.0) tensor var_253 = sub(x = var_18, y = use_roll_1)[name = tensor("op_253")]; tensor var_254 = mul(x = filled_window_1, y = var_253)[name = tensor("op_254")]; tensor var_255 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_255")]; - tensor input_21 = add(x = var_254, y = var_255)[name = tensor("input_21")]; + tensor window_3 = add(x = var_254, y = var_255)[name = tensor("window_3")]; tensor var_257 = const()[name = tensor("op_257"), val = tensor(0x1p+0)]; tensor var_258 = add(x = filled_1, y = var_257)[name = tensor("op_258")]; tensor clip_1 = clip(alpha = var_22, beta = var_17, x = var_258)[name = tensor("clip_1")]; + tensor var_261_begin_0 = const()[name = tensor("op_261_begin_0"), val = tensor([0, 1, 0])]; + tensor var_261_end_0 = const()[name = tensor("op_261_end_0"), val = tensor([1, 2, 256])]; + tensor var_261_end_mask_0 = const()[name = tensor("op_261_end_mask_0"), val = tensor([true, false, true])]; + tensor var_261 = slice_by_index(begin = var_261_begin_0, end = var_261_end_0, end_mask = var_261_end_mask_0, x = x_3)[name = tensor("op_261")]; + tensor var_264_begin_0 = const()[name = tensor("op_264_begin_0"), val = tensor([0, 1, 0])]; + tensor var_264_end_0 = const()[name = tensor("op_264_end_0"), val = tensor([1, 16, 256])]; + tensor var_264_end_mask_0 = const()[name = tensor("op_264_end_mask_0"), val = tensor([true, true, true])]; + tensor var_264 = slice_by_index(begin = var_264_begin_0, end = var_264_end_0, end_mask = var_264_end_mask_0, x = window_3)[name = tensor("op_264")]; + tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; + tensor rolled_3 = concat(axis = var_34, interleave = rolled_3_interleave_0, values = (var_264, var_261))[name = tensor("rolled_3")]; + tensor clip_2 = clip(alpha = var_22, beta = var_21, x = clip_1)[name = tensor("clip_2")]; + tensor var_269 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_269")]; + tensor var_270_dtype_0 = const()[name = tensor("op_270_dtype_0"), val = tensor("fp32")]; + tensor var_271 = const()[name = tensor("op_271"), val = tensor([1, 16, 1])]; + tensor var_270 = cast(dtype = var_270_dtype_0, x = var_269)[name = tensor("cast_189")]; + tensor fill_mask_3 = reshape(shape = var_271, x = var_270)[name = tensor("fill_mask_3")]; + tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_261)[name = tensor("x_expanded_3")]; + tensor var_275 = sub(x = var_18, y = fill_mask_3)[name = tensor("op_275")]; + tensor var_276 = mul(x = window_3, y = var_275)[name = tensor("op_276")]; + tensor var_277 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_277")]; + tensor filled_window_3 = add(x = var_276, y = var_277)[name = tensor("filled_window_3")]; + tensor var_279 = greater_equal(x = clip_1, y = var_17)[name = tensor("op_279")]; + tensor var_280_dtype_0 = const()[name = tensor("op_280_dtype_0"), val = tensor("fp32")]; + tensor var_281 = const()[name = tensor("op_281"), val = tensor([1, 1, 1])]; + tensor var_280 = cast(dtype = var_280_dtype_0, x = var_279)[name = tensor("cast_188")]; + tensor use_roll_3 = reshape(shape = var_281, x = var_280)[name = tensor("use_roll_3")]; + tensor var_283 = sub(x = var_18, y = use_roll_3)[name = tensor("op_283")]; + tensor var_284 = mul(x = filled_window_3, y = var_283)[name = tensor("op_284")]; + tensor var_285 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_285")]; + tensor window_5 = add(x = var_284, y = var_285)[name = tensor("window_5")]; + tensor var_287 = const()[name = tensor("op_287"), val = tensor(0x1p+0)]; + tensor var_288 = add(x = clip_1, y = var_287)[name = tensor("op_288")]; + tensor clip_3 = clip(alpha = var_22, beta = var_17, x = var_288)[name = tensor("clip_3")]; + tensor var_291_begin_0 = const()[name = tensor("op_291_begin_0"), val = tensor([0, 2, 0])]; + tensor var_291_end_0 = const()[name = tensor("op_291_end_0"), val = tensor([1, 3, 256])]; + tensor var_291_end_mask_0 = const()[name = tensor("op_291_end_mask_0"), val = tensor([true, false, true])]; + tensor var_291 = slice_by_index(begin = var_291_begin_0, end = var_291_end_0, end_mask = var_291_end_mask_0, x = x_3)[name = tensor("op_291")]; + tensor var_294_begin_0 = const()[name = tensor("op_294_begin_0"), val = tensor([0, 1, 0])]; + tensor var_294_end_0 = const()[name = tensor("op_294_end_0"), val = tensor([1, 16, 256])]; + tensor var_294_end_mask_0 = const()[name = tensor("op_294_end_mask_0"), val = tensor([true, true, true])]; + tensor var_294 = slice_by_index(begin = var_294_begin_0, end = var_294_end_0, end_mask = var_294_end_mask_0, x = window_5)[name = tensor("op_294")]; + tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; + tensor rolled_5 = concat(axis = var_34, interleave = rolled_5_interleave_0, values = (var_294, var_291))[name = tensor("rolled_5")]; + tensor clip_4 = clip(alpha = var_22, beta = var_21, x = clip_3)[name = tensor("clip_4")]; + tensor var_299 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_299")]; + tensor var_300_dtype_0 = const()[name = tensor("op_300_dtype_0"), val = tensor("fp32")]; + tensor var_301 = const()[name = tensor("op_301"), val = tensor([1, 16, 1])]; + tensor var_300 = cast(dtype = var_300_dtype_0, x = var_299)[name = tensor("cast_187")]; + tensor fill_mask_5 = reshape(shape = var_301, x = var_300)[name = tensor("fill_mask_5")]; + tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_291)[name = tensor("x_expanded_5")]; + tensor var_305 = sub(x = var_18, y = fill_mask_5)[name = tensor("op_305")]; + tensor var_306 = mul(x = window_5, y = var_305)[name = tensor("op_306")]; + tensor var_307 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_307")]; + tensor filled_window_5 = add(x = var_306, y = var_307)[name = tensor("filled_window_5")]; + tensor var_309 = greater_equal(x = clip_3, y = var_17)[name = tensor("op_309")]; + tensor var_310_dtype_0 = const()[name = tensor("op_310_dtype_0"), val = tensor("fp32")]; + tensor var_311 = const()[name = tensor("op_311"), val = tensor([1, 1, 1])]; + tensor var_310 = cast(dtype = var_310_dtype_0, x = var_309)[name = tensor("cast_186")]; + tensor use_roll_5 = reshape(shape = var_311, x = var_310)[name = tensor("use_roll_5")]; + tensor var_313 = sub(x = var_18, y = use_roll_5)[name = tensor("op_313")]; + tensor var_314 = mul(x = filled_window_5, y = var_313)[name = tensor("op_314")]; + tensor var_315 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_315")]; + tensor window_7 = add(x = var_314, y = var_315)[name = tensor("window_7")]; + tensor var_317 = const()[name = tensor("op_317"), val = tensor(0x1p+0)]; + tensor var_318 = add(x = clip_3, y = var_317)[name = tensor("op_318")]; + tensor clip_5 = clip(alpha = var_22, beta = var_17, x = var_318)[name = tensor("clip_5")]; + tensor var_321_begin_0 = const()[name = tensor("op_321_begin_0"), val = tensor([0, 3, 0])]; + tensor var_321_end_0 = const()[name = tensor("op_321_end_0"), val = tensor([1, 4, 256])]; + tensor var_321_end_mask_0 = const()[name = tensor("op_321_end_mask_0"), val = tensor([true, false, true])]; + tensor var_321 = slice_by_index(begin = var_321_begin_0, end = var_321_end_0, end_mask = var_321_end_mask_0, x = x_3)[name = tensor("op_321")]; + tensor var_324_begin_0 = const()[name = tensor("op_324_begin_0"), val = tensor([0, 1, 0])]; + tensor var_324_end_0 = const()[name = tensor("op_324_end_0"), val = tensor([1, 16, 256])]; + tensor var_324_end_mask_0 = const()[name = tensor("op_324_end_mask_0"), val = tensor([true, true, true])]; + tensor var_324 = slice_by_index(begin = var_324_begin_0, end = var_324_end_0, end_mask = var_324_end_mask_0, x = window_7)[name = tensor("op_324")]; + tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; + tensor rolled_7 = concat(axis = var_34, interleave = rolled_7_interleave_0, values = (var_324, var_321))[name = tensor("rolled_7")]; + tensor clip_6 = clip(alpha = var_22, beta = var_21, x = clip_5)[name = tensor("clip_6")]; + tensor var_329 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_329")]; + tensor var_330_dtype_0 = const()[name = tensor("op_330_dtype_0"), val = tensor("fp32")]; + tensor var_331 = const()[name = tensor("op_331"), val = tensor([1, 16, 1])]; + tensor var_330 = cast(dtype = var_330_dtype_0, x = var_329)[name = tensor("cast_185")]; + tensor fill_mask_7 = reshape(shape = var_331, x = var_330)[name = tensor("fill_mask_7")]; + tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_321)[name = tensor("x_expanded_7")]; + tensor var_335 = sub(x = var_18, y = fill_mask_7)[name = tensor("op_335")]; + tensor var_336 = mul(x = window_7, y = var_335)[name = tensor("op_336")]; + tensor var_337 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_337")]; + tensor filled_window_7 = add(x = var_336, y = var_337)[name = tensor("filled_window_7")]; + tensor var_339 = greater_equal(x = clip_5, y = var_17)[name = tensor("op_339")]; + tensor var_340_dtype_0 = const()[name = tensor("op_340_dtype_0"), val = tensor("fp32")]; + tensor var_341 = const()[name = tensor("op_341"), val = tensor([1, 1, 1])]; + tensor var_340 = cast(dtype = var_340_dtype_0, x = var_339)[name = tensor("cast_184")]; + tensor use_roll_7 = reshape(shape = var_341, x = var_340)[name = tensor("use_roll_7")]; + tensor var_343 = sub(x = var_18, y = use_roll_7)[name = tensor("op_343")]; + tensor var_344 = mul(x = filled_window_7, y = var_343)[name = tensor("op_344")]; + tensor var_345 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_345")]; + tensor window_9 = add(x = var_344, y = var_345)[name = tensor("window_9")]; + tensor var_347 = const()[name = tensor("op_347"), val = tensor(0x1p+0)]; + tensor var_348 = add(x = clip_5, y = var_347)[name = tensor("op_348")]; + tensor clip_7 = clip(alpha = var_22, beta = var_17, x = var_348)[name = tensor("clip_7")]; + tensor var_351_begin_0 = const()[name = tensor("op_351_begin_0"), val = tensor([0, 4, 0])]; + tensor var_351_end_0 = const()[name = tensor("op_351_end_0"), val = tensor([1, 1, 256])]; + tensor var_351_end_mask_0 = const()[name = tensor("op_351_end_mask_0"), val = tensor([true, true, true])]; + tensor var_351 = slice_by_index(begin = var_351_begin_0, end = var_351_end_0, end_mask = var_351_end_mask_0, x = x_3)[name = tensor("op_351")]; + tensor var_354_begin_0 = const()[name = tensor("op_354_begin_0"), val = tensor([0, 1, 0])]; + tensor var_354_end_0 = const()[name = tensor("op_354_end_0"), val = tensor([1, 16, 256])]; + tensor var_354_end_mask_0 = const()[name = tensor("op_354_end_mask_0"), val = tensor([true, true, true])]; + tensor var_354 = slice_by_index(begin = var_354_begin_0, end = var_354_end_0, end_mask = var_354_end_mask_0, x = window_9)[name = tensor("op_354")]; + tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; + tensor rolled_9 = concat(axis = var_34, interleave = rolled_9_interleave_0, values = (var_354, var_351))[name = tensor("rolled_9")]; + tensor clip_8 = clip(alpha = var_22, beta = var_21, x = clip_7)[name = tensor("clip_8")]; + tensor var_359 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_359")]; + tensor var_360_dtype_0 = const()[name = tensor("op_360_dtype_0"), val = tensor("fp32")]; + tensor var_361 = const()[name = tensor("op_361"), val = tensor([1, 16, 1])]; + tensor var_360 = cast(dtype = var_360_dtype_0, x = var_359)[name = tensor("cast_183")]; + tensor fill_mask_9 = reshape(shape = var_361, x = var_360)[name = tensor("fill_mask_9")]; + tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_351)[name = tensor("x_expanded_9")]; + tensor var_365 = sub(x = var_18, y = fill_mask_9)[name = tensor("op_365")]; + tensor var_366 = mul(x = window_9, y = var_365)[name = tensor("op_366")]; + tensor var_367 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_367")]; + tensor filled_window_9 = add(x = var_366, y = var_367)[name = tensor("filled_window_9")]; + tensor var_369 = greater_equal(x = clip_7, y = var_17)[name = tensor("op_369")]; + tensor var_370_dtype_0 = const()[name = tensor("op_370_dtype_0"), val = tensor("fp32")]; + tensor var_371 = const()[name = tensor("op_371"), val = tensor([1, 1, 1])]; + tensor var_370 = cast(dtype = var_370_dtype_0, x = var_369)[name = tensor("cast_182")]; + tensor use_roll_9 = reshape(shape = var_371, x = var_370)[name = tensor("use_roll_9")]; + tensor var_373 = sub(x = var_18, y = use_roll_9)[name = tensor("op_373")]; + tensor var_374 = mul(x = filled_window_9, y = var_373)[name = tensor("op_374")]; + tensor var_375 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_375")]; + tensor window_11 = add(x = var_374, y = var_375)[name = tensor("window_11")]; + tensor var_377 = const()[name = tensor("op_377"), val = tensor(0x1p+0)]; + tensor var_378 = add(x = clip_7, y = var_377)[name = tensor("op_378")]; + tensor clip_9 = clip(alpha = var_22, beta = var_17, x = var_378)[name = tensor("clip_9")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_31, interleave = input_21_interleave_0, values = (window_3, window_5, window_7, window_9, window_11))[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; - tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; + tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; tensor inputs_3_pad_type_0 = const()[name = tensor("inputs_3_pad_type_0"), val = tensor("valid")]; tensor inputs_3_strides_0 = const()[name = tensor("inputs_3_strides_0"), val = tensor([1])]; tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_82")]; - tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_279_split_sizes_0 = const()[name = tensor("op_279_split_sizes_0"), val = tensor([256, 256])]; - tensor var_279_axis_0 = const()[name = tensor("op_279_axis_0"), val = tensor(1)]; - tensor var_279_0, tensor var_279_1 = split(axis = var_279_axis_0, split_sizes = var_279_split_sizes_0, x = inputs_3)[name = tensor("op_279")]; - tensor var_281 = sigmoid(x = var_279_1)[name = tensor("op_281")]; - tensor inputs_5 = mul(x = var_279_0, y = var_281)[name = tensor("inputs_5")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; + tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; + tensor var_401_split_sizes_0 = const()[name = tensor("op_401_split_sizes_0"), val = tensor([256, 256])]; + tensor var_401_axis_0 = const()[name = tensor("op_401_axis_0"), val = tensor(1)]; + tensor var_401_0, tensor var_401_1 = split(axis = var_401_axis_0, split_sizes = var_401_split_sizes_0, x = inputs_3)[name = tensor("op_401")]; + tensor var_403 = sigmoid(x = var_401_1)[name = tensor("op_403")]; + tensor inputs_5 = mul(x = var_401_0, y = var_403)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; tensor outputs_aug_1_strides_0 = const()[name = tensor("outputs_aug_1_strides_0"), val = tensor([1])]; tensor outputs_aug_1_dilations_0 = const()[name = tensor("outputs_aug_1_dilations_0"), val = tensor([1])]; - tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; + tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; tensor input_25_begin_0 = const()[name = tensor("input_25_begin_0"), val = tensor([0, 0, 0])]; - tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([1, 256, 16])]; + tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([5, 256, 16])]; tensor input_25_end_mask_0 = const()[name = tensor("input_25_end_mask_0"), val = tensor([true, true, false])]; - tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; - tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; - tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; + tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; + tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; + tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; tensor input_29_pad_type_0 = const()[name = tensor("input_29_pad_type_0"), val = tensor("valid")]; tensor input_29_strides_0 = const()[name = tensor("input_29_strides_0"), val = tensor([1])]; tensor input_29_pad_0 = const()[name = tensor("input_29_pad_0"), val = tensor([0, 0])]; tensor input_29_dilations_0 = const()[name = tensor("input_29_dilations_0"), val = tensor([1])]; tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; - tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; + tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_312_begin_0 = const()[name = tensor("op_312_begin_0"), val = tensor([0, -1, 0])]; - tensor var_312_end_0 = const()[name = tensor("op_312_end_0"), val = tensor([1, 16, 256])]; - tensor var_312_end_mask_0 = const()[name = tensor("op_312_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_81")]; - tensor var_312 = slice_by_index(begin = var_312_begin_0, end = var_312_end_0, end_mask = var_312_end_mask_0, x = conv_out_1)[name = tensor("op_312")]; - tensor var_315_begin_0 = const()[name = tensor("op_315_begin_0"), val = tensor([0, 1, 0])]; - tensor var_315_end_0 = const()[name = tensor("op_315_end_0"), val = tensor([1, 2, 256])]; - tensor var_315_end_mask_0 = const()[name = tensor("op_315_end_mask_0"), val = tensor([true, false, true])]; - tensor var_315 = slice_by_index(begin = var_315_begin_0, end = var_315_end_0, end_mask = var_315_end_mask_0, x = x_3)[name = tensor("op_315")]; - tensor var_318_begin_0 = const()[name = tensor("op_318_begin_0"), val = tensor([0, 1, 0])]; - tensor var_318_end_0 = const()[name = tensor("op_318_end_0"), val = tensor([1, 16, 256])]; - tensor var_318_end_mask_0 = const()[name = tensor("op_318_end_mask_0"), val = tensor([true, true, true])]; - tensor var_318 = slice_by_index(begin = var_318_begin_0, end = var_318_end_0, end_mask = var_318_end_mask_0, x = input_21)[name = tensor("op_318")]; - tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_34, interleave = rolled_3_interleave_0, values = (var_318, var_315))[name = tensor("rolled_3")]; - tensor clip_2 = clip(alpha = var_22, beta = var_21, x = clip_1)[name = tensor("clip_2")]; - tensor var_323 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_323")]; - tensor var_324_dtype_0 = const()[name = tensor("op_324_dtype_0"), val = tensor("fp32")]; - tensor var_325 = const()[name = tensor("op_325"), val = tensor([1, 16, 1])]; - tensor var_324 = cast(dtype = var_324_dtype_0, x = var_323)[name = tensor("cast_189")]; - tensor fill_mask_3 = reshape(shape = var_325, x = var_324)[name = tensor("fill_mask_3")]; - tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_315)[name = tensor("x_expanded_3")]; - tensor var_329 = sub(x = var_18, y = fill_mask_3)[name = tensor("op_329")]; - tensor var_330 = mul(x = input_21, y = var_329)[name = tensor("op_330")]; - tensor var_331 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_331")]; - tensor filled_window_3 = add(x = var_330, y = var_331)[name = tensor("filled_window_3")]; - tensor var_333 = greater_equal(x = clip_1, y = var_17)[name = tensor("op_333")]; - tensor var_334_dtype_0 = const()[name = tensor("op_334_dtype_0"), val = tensor("fp32")]; - tensor var_335 = const()[name = tensor("op_335"), val = tensor([1, 1, 1])]; - tensor var_334 = cast(dtype = var_334_dtype_0, x = var_333)[name = tensor("cast_188")]; - tensor use_roll_3 = reshape(shape = var_335, x = var_334)[name = tensor("use_roll_3")]; - tensor var_337 = sub(x = var_18, y = use_roll_3)[name = tensor("op_337")]; - tensor var_338 = mul(x = filled_window_3, y = var_337)[name = tensor("op_338")]; - tensor var_339 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_339")]; - tensor input_31 = add(x = var_338, y = var_339)[name = tensor("input_31")]; - tensor var_341 = const()[name = tensor("op_341"), val = tensor(0x1p+0)]; - tensor var_342 = add(x = clip_1, y = var_341)[name = tensor("op_342")]; - tensor clip_3 = clip(alpha = var_22, beta = var_17, x = var_342)[name = tensor("clip_3")]; + tensor var_434_begin_0 = const()[name = tensor("op_434_begin_0"), val = tensor([0, -1, 0])]; + tensor var_434_end_0 = const()[name = tensor("op_434_end_0"), val = tensor([5, 16, 256])]; + tensor var_434_end_mask_0 = const()[name = tensor("op_434_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_434 = slice_by_index(begin = var_434_begin_0, end = var_434_end_0, end_mask = var_434_end_mask_0, x = conv_out_1)[name = tensor("op_434")]; + tensor var_436_perm_0 = const()[name = tensor("op_436_perm_0"), val = tensor([1, 0, 2])]; + tensor var_436 = transpose(perm = var_436_perm_0, x = var_434)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_436)[name = tensor("input_31")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; + tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; + tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; + tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; + tensor var_459 = const()[name = tensor("op_459"), val = tensor(0x1p-1)]; + tensor var_460 = mul(x = input_39, y = var_459)[name = tensor("op_460")]; + tensor input_41 = add(x = var_460, y = input_31)[name = tensor("input_41")]; + tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; + tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_36, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; + tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; + tensor input_45 = layer_norm(axes = input_45_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_43)[name = tensor("input_45")]; + tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; + tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; + tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; + tensor var_489 = const()[name = tensor("op_489"), val = tensor(0x1p-1)]; + tensor var_490 = mul(x = input_51, y = var_489)[name = tensor("op_490")]; + tensor input_53 = add(x = var_490, y = input_43)[name = tensor("input_53")]; tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; - tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_31)[name = tensor("x_7")]; - tensor input_33_perm_0 = const()[name = tensor("input_33_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_9_pad_type_0 = const()[name = tensor("inputs_9_pad_type_0"), val = tensor("valid")]; - tensor inputs_9_strides_0 = const()[name = tensor("inputs_9_strides_0"), val = tensor([1])]; - tensor inputs_9_pad_0 = const()[name = tensor("inputs_9_pad_0"), val = tensor([0, 0])]; - tensor inputs_9_dilations_0 = const()[name = tensor("inputs_9_dilations_0"), val = tensor([1])]; - tensor inputs_9_groups_0 = const()[name = tensor("inputs_9_groups_0"), val = tensor(1)]; - tensor input_33 = transpose(perm = input_33_perm_0, x = x_7)[name = tensor("transpose_80")]; - tensor inputs_9 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_9_dilations_0, groups = inputs_9_groups_0, pad = inputs_9_pad_0, pad_type = inputs_9_pad_type_0, strides = inputs_9_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_33)[name = tensor("inputs_9")]; - tensor var_363_split_sizes_0 = const()[name = tensor("op_363_split_sizes_0"), val = tensor([256, 256])]; - tensor var_363_axis_0 = const()[name = tensor("op_363_axis_0"), val = tensor(1)]; - tensor var_363_0, tensor var_363_1 = split(axis = var_363_axis_0, split_sizes = var_363_split_sizes_0, x = inputs_9)[name = tensor("op_363")]; - tensor var_365 = sigmoid(x = var_363_1)[name = tensor("op_365")]; - tensor inputs_11 = mul(x = var_363_0, y = var_365)[name = tensor("inputs_11")]; - tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; - tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; - tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; - tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_11)[name = tensor("outputs_aug_3")]; - tensor input_35_begin_0 = const()[name = tensor("input_35_begin_0"), val = tensor([0, 0, 0])]; - tensor input_35_end_0 = const()[name = tensor("input_35_end_0"), val = tensor([1, 256, 16])]; - tensor input_35_end_mask_0 = const()[name = tensor("input_35_end_mask_0"), val = tensor([true, true, false])]; - tensor input_35 = slice_by_index(begin = input_35_begin_0, end = input_35_end_0, end_mask = input_35_end_mask_0, x = outputs_aug_3)[name = tensor("input_35")]; - tensor inputs_13 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_35)[name = tensor("inputs_13")]; - tensor input_37 = silu(x = inputs_13)[name = tensor("input_37")]; - tensor input_39_pad_type_0 = const()[name = tensor("input_39_pad_type_0"), val = tensor("valid")]; - tensor input_39_strides_0 = const()[name = tensor("input_39_strides_0"), val = tensor([1])]; - tensor input_39_pad_0 = const()[name = tensor("input_39_pad_0"), val = tensor([0, 0])]; - tensor input_39_dilations_0 = const()[name = tensor("input_39_dilations_0"), val = tensor([1])]; - tensor input_39_groups_0 = const()[name = tensor("input_39_groups_0"), val = tensor(1)]; - tensor input_39 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_37)[name = tensor("input_39")]; - tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_396_begin_0 = const()[name = tensor("op_396_begin_0"), val = tensor([0, -1, 0])]; - tensor var_396_end_0 = const()[name = tensor("op_396_end_0"), val = tensor([1, 16, 256])]; - tensor var_396_end_mask_0 = const()[name = tensor("op_396_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_39)[name = tensor("transpose_79")]; - tensor var_396 = slice_by_index(begin = var_396_begin_0, end = var_396_end_0, end_mask = var_396_end_mask_0, x = conv_out_3)[name = tensor("op_396")]; - tensor var_399_begin_0 = const()[name = tensor("op_399_begin_0"), val = tensor([0, 2, 0])]; - tensor var_399_end_0 = const()[name = tensor("op_399_end_0"), val = tensor([1, 3, 256])]; - tensor var_399_end_mask_0 = const()[name = tensor("op_399_end_mask_0"), val = tensor([true, false, true])]; - tensor var_399 = slice_by_index(begin = var_399_begin_0, end = var_399_end_0, end_mask = var_399_end_mask_0, x = x_3)[name = tensor("op_399")]; - tensor var_402_begin_0 = const()[name = tensor("op_402_begin_0"), val = tensor([0, 1, 0])]; - tensor var_402_end_0 = const()[name = tensor("op_402_end_0"), val = tensor([1, 16, 256])]; - tensor var_402_end_mask_0 = const()[name = tensor("op_402_end_mask_0"), val = tensor([true, true, true])]; - tensor var_402 = slice_by_index(begin = var_402_begin_0, end = var_402_end_0, end_mask = var_402_end_mask_0, x = input_31)[name = tensor("op_402")]; - tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_34, interleave = rolled_5_interleave_0, values = (var_402, var_399))[name = tensor("rolled_5")]; - tensor clip_4 = clip(alpha = var_22, beta = var_21, x = clip_3)[name = tensor("clip_4")]; - tensor var_407 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_407")]; - tensor var_408_dtype_0 = const()[name = tensor("op_408_dtype_0"), val = tensor("fp32")]; - tensor var_409 = const()[name = tensor("op_409"), val = tensor([1, 16, 1])]; - tensor var_408 = cast(dtype = var_408_dtype_0, x = var_407)[name = tensor("cast_187")]; - tensor fill_mask_5 = reshape(shape = var_409, x = var_408)[name = tensor("fill_mask_5")]; - tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_399)[name = tensor("x_expanded_5")]; - tensor var_413 = sub(x = var_18, y = fill_mask_5)[name = tensor("op_413")]; - tensor var_414 = mul(x = input_31, y = var_413)[name = tensor("op_414")]; - tensor var_415 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_415")]; - tensor filled_window_5 = add(x = var_414, y = var_415)[name = tensor("filled_window_5")]; - tensor var_417 = greater_equal(x = clip_3, y = var_17)[name = tensor("op_417")]; - tensor var_418_dtype_0 = const()[name = tensor("op_418_dtype_0"), val = tensor("fp32")]; - tensor var_419 = const()[name = tensor("op_419"), val = tensor([1, 1, 1])]; - tensor var_418 = cast(dtype = var_418_dtype_0, x = var_417)[name = tensor("cast_186")]; - tensor use_roll_5 = reshape(shape = var_419, x = var_418)[name = tensor("use_roll_5")]; - tensor var_421 = sub(x = var_18, y = use_roll_5)[name = tensor("op_421")]; - tensor var_422 = mul(x = filled_window_5, y = var_421)[name = tensor("op_422")]; - tensor var_423 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_423")]; - tensor input_41 = add(x = var_422, y = var_423)[name = tensor("input_41")]; - tensor var_425 = const()[name = tensor("op_425"), val = tensor(0x1p+0)]; - tensor var_426 = add(x = clip_3, y = var_425)[name = tensor("op_426")]; - tensor clip_5 = clip(alpha = var_22, beta = var_17, x = var_426)[name = tensor("clip_5")]; - tensor x_9_axes_0 = const()[name = tensor("x_9_axes_0"), val = tensor([-1])]; - tensor x_9 = layer_norm(axes = x_9_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_41)[name = tensor("x_9")]; - tensor input_43_perm_0 = const()[name = tensor("input_43_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_15_pad_type_0 = const()[name = tensor("inputs_15_pad_type_0"), val = tensor("valid")]; - tensor inputs_15_strides_0 = const()[name = tensor("inputs_15_strides_0"), val = tensor([1])]; - tensor inputs_15_pad_0 = const()[name = tensor("inputs_15_pad_0"), val = tensor([0, 0])]; - tensor inputs_15_dilations_0 = const()[name = tensor("inputs_15_dilations_0"), val = tensor([1])]; - tensor inputs_15_groups_0 = const()[name = tensor("inputs_15_groups_0"), val = tensor(1)]; - tensor input_43 = transpose(perm = input_43_perm_0, x = x_9)[name = tensor("transpose_78")]; - tensor inputs_15 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_15_dilations_0, groups = inputs_15_groups_0, pad = inputs_15_pad_0, pad_type = inputs_15_pad_type_0, strides = inputs_15_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_43)[name = tensor("inputs_15")]; - tensor var_447_split_sizes_0 = const()[name = tensor("op_447_split_sizes_0"), val = tensor([256, 256])]; - tensor var_447_axis_0 = const()[name = tensor("op_447_axis_0"), val = tensor(1)]; - tensor var_447_0, tensor var_447_1 = split(axis = var_447_axis_0, split_sizes = var_447_split_sizes_0, x = inputs_15)[name = tensor("op_447")]; - tensor var_449 = sigmoid(x = var_447_1)[name = tensor("op_449")]; - tensor inputs_17 = mul(x = var_447_0, y = var_449)[name = tensor("inputs_17")]; - tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; - tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; - tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; - tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_17)[name = tensor("outputs_aug_5")]; - tensor input_45_begin_0 = const()[name = tensor("input_45_begin_0"), val = tensor([0, 0, 0])]; - tensor input_45_end_0 = const()[name = tensor("input_45_end_0"), val = tensor([1, 256, 16])]; - tensor input_45_end_mask_0 = const()[name = tensor("input_45_end_mask_0"), val = tensor([true, true, false])]; - tensor input_45 = slice_by_index(begin = input_45_begin_0, end = input_45_end_0, end_mask = input_45_end_mask_0, x = outputs_aug_5)[name = tensor("input_45")]; - tensor inputs_19 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_45)[name = tensor("inputs_19")]; - tensor input_47 = silu(x = inputs_19)[name = tensor("input_47")]; - tensor input_49_pad_type_0 = const()[name = tensor("input_49_pad_type_0"), val = tensor("valid")]; - tensor input_49_strides_0 = const()[name = tensor("input_49_strides_0"), val = tensor([1])]; - tensor input_49_pad_0 = const()[name = tensor("input_49_pad_0"), val = tensor([0, 0])]; - tensor input_49_dilations_0 = const()[name = tensor("input_49_dilations_0"), val = tensor([1])]; - tensor input_49_groups_0 = const()[name = tensor("input_49_groups_0"), val = tensor(1)]; - tensor input_49 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_49_dilations_0, groups = input_49_groups_0, pad = input_49_pad_0, pad_type = input_49_pad_type_0, strides = input_49_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_47)[name = tensor("input_49")]; - tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_480_begin_0 = const()[name = tensor("op_480_begin_0"), val = tensor([0, -1, 0])]; - tensor var_480_end_0 = const()[name = tensor("op_480_end_0"), val = tensor([1, 16, 256])]; - tensor var_480_end_mask_0 = const()[name = tensor("op_480_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_49)[name = tensor("transpose_77")]; - tensor var_480 = slice_by_index(begin = var_480_begin_0, end = var_480_end_0, end_mask = var_480_end_mask_0, x = conv_out_5)[name = tensor("op_480")]; - tensor var_483_begin_0 = const()[name = tensor("op_483_begin_0"), val = tensor([0, 3, 0])]; - tensor var_483_end_0 = const()[name = tensor("op_483_end_0"), val = tensor([1, 4, 256])]; - tensor var_483_end_mask_0 = const()[name = tensor("op_483_end_mask_0"), val = tensor([true, false, true])]; - tensor var_483 = slice_by_index(begin = var_483_begin_0, end = var_483_end_0, end_mask = var_483_end_mask_0, x = x_3)[name = tensor("op_483")]; - tensor var_486_begin_0 = const()[name = tensor("op_486_begin_0"), val = tensor([0, 1, 0])]; - tensor var_486_end_0 = const()[name = tensor("op_486_end_0"), val = tensor([1, 16, 256])]; - tensor var_486_end_mask_0 = const()[name = tensor("op_486_end_mask_0"), val = tensor([true, true, true])]; - tensor var_486 = slice_by_index(begin = var_486_begin_0, end = var_486_end_0, end_mask = var_486_end_mask_0, x = input_41)[name = tensor("op_486")]; - tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; - tensor rolled_7 = concat(axis = var_34, interleave = rolled_7_interleave_0, values = (var_486, var_483))[name = tensor("rolled_7")]; - tensor clip_6 = clip(alpha = var_22, beta = var_21, x = clip_5)[name = tensor("clip_6")]; - tensor var_491 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_491")]; - tensor var_492_dtype_0 = const()[name = tensor("op_492_dtype_0"), val = tensor("fp32")]; - tensor var_493 = const()[name = tensor("op_493"), val = tensor([1, 16, 1])]; - tensor var_492 = cast(dtype = var_492_dtype_0, x = var_491)[name = tensor("cast_185")]; - tensor fill_mask_7 = reshape(shape = var_493, x = var_492)[name = tensor("fill_mask_7")]; - tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_483)[name = tensor("x_expanded_7")]; - tensor var_497 = sub(x = var_18, y = fill_mask_7)[name = tensor("op_497")]; - tensor var_498 = mul(x = input_41, y = var_497)[name = tensor("op_498")]; - tensor var_499 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_499")]; - tensor filled_window_7 = add(x = var_498, y = var_499)[name = tensor("filled_window_7")]; - tensor var_501 = greater_equal(x = clip_5, y = var_17)[name = tensor("op_501")]; - tensor var_502_dtype_0 = const()[name = tensor("op_502_dtype_0"), val = tensor("fp32")]; - tensor var_503 = const()[name = tensor("op_503"), val = tensor([1, 1, 1])]; - tensor var_502 = cast(dtype = var_502_dtype_0, x = var_501)[name = tensor("cast_184")]; - tensor use_roll_7 = reshape(shape = var_503, x = var_502)[name = tensor("use_roll_7")]; - tensor var_505 = sub(x = var_18, y = use_roll_7)[name = tensor("op_505")]; - tensor var_506 = mul(x = filled_window_7, y = var_505)[name = tensor("op_506")]; - tensor var_507 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_507")]; - tensor input_51 = add(x = var_506, y = var_507)[name = tensor("input_51")]; - tensor var_509 = const()[name = tensor("op_509"), val = tensor(0x1p+0)]; - tensor var_510 = add(x = clip_5, y = var_509)[name = tensor("op_510")]; - tensor clip_7 = clip(alpha = var_22, beta = var_17, x = var_510)[name = tensor("clip_7")]; - tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; - tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_51)[name = tensor("x_11")]; - tensor input_53_perm_0 = const()[name = tensor("input_53_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_21_pad_type_0 = const()[name = tensor("inputs_21_pad_type_0"), val = tensor("valid")]; - tensor inputs_21_strides_0 = const()[name = tensor("inputs_21_strides_0"), val = tensor([1])]; - tensor inputs_21_pad_0 = const()[name = tensor("inputs_21_pad_0"), val = tensor([0, 0])]; - tensor inputs_21_dilations_0 = const()[name = tensor("inputs_21_dilations_0"), val = tensor([1])]; - tensor inputs_21_groups_0 = const()[name = tensor("inputs_21_groups_0"), val = tensor(1)]; - tensor input_53 = transpose(perm = input_53_perm_0, x = x_11)[name = tensor("transpose_76")]; - tensor inputs_21 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_21_dilations_0, groups = inputs_21_groups_0, pad = inputs_21_pad_0, pad_type = inputs_21_pad_type_0, strides = inputs_21_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_53)[name = tensor("inputs_21")]; - tensor var_531_split_sizes_0 = const()[name = tensor("op_531_split_sizes_0"), val = tensor([256, 256])]; - tensor var_531_axis_0 = const()[name = tensor("op_531_axis_0"), val = tensor(1)]; - tensor var_531_0, tensor var_531_1 = split(axis = var_531_axis_0, split_sizes = var_531_split_sizes_0, x = inputs_21)[name = tensor("op_531")]; - tensor var_533 = sigmoid(x = var_531_1)[name = tensor("op_533")]; - tensor inputs_23 = mul(x = var_531_0, y = var_533)[name = tensor("inputs_23")]; - tensor outputs_aug_7_pad_type_0 = const()[name = tensor("outputs_aug_7_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_7_pad_0 = const()[name = tensor("outputs_aug_7_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_7_groups_0 = const()[name = tensor("outputs_aug_7_groups_0"), val = tensor(256)]; - tensor outputs_aug_7_strides_0 = const()[name = tensor("outputs_aug_7_strides_0"), val = tensor([1])]; - tensor outputs_aug_7_dilations_0 = const()[name = tensor("outputs_aug_7_dilations_0"), val = tensor([1])]; - tensor outputs_aug_7 = conv(dilations = outputs_aug_7_dilations_0, groups = outputs_aug_7_groups_0, pad = outputs_aug_7_pad_0, pad_type = outputs_aug_7_pad_type_0, strides = outputs_aug_7_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_23)[name = tensor("outputs_aug_7")]; - tensor input_55_begin_0 = const()[name = tensor("input_55_begin_0"), val = tensor([0, 0, 0])]; - tensor input_55_end_0 = const()[name = tensor("input_55_end_0"), val = tensor([1, 256, 16])]; - tensor input_55_end_mask_0 = const()[name = tensor("input_55_end_mask_0"), val = tensor([true, true, false])]; - tensor input_55 = slice_by_index(begin = input_55_begin_0, end = input_55_end_0, end_mask = input_55_end_mask_0, x = outputs_aug_7)[name = tensor("input_55")]; - tensor inputs_25 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_55)[name = tensor("inputs_25")]; - tensor input_57 = silu(x = inputs_25)[name = tensor("input_57")]; - tensor input_59_pad_type_0 = const()[name = tensor("input_59_pad_type_0"), val = tensor("valid")]; - tensor input_59_strides_0 = const()[name = tensor("input_59_strides_0"), val = tensor([1])]; - tensor input_59_pad_0 = const()[name = tensor("input_59_pad_0"), val = tensor([0, 0])]; - tensor input_59_dilations_0 = const()[name = tensor("input_59_dilations_0"), val = tensor([1])]; - tensor input_59_groups_0 = const()[name = tensor("input_59_groups_0"), val = tensor(1)]; - tensor input_59 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_59_dilations_0, groups = input_59_groups_0, pad = input_59_pad_0, pad_type = input_59_pad_type_0, strides = input_59_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_57)[name = tensor("input_59")]; - tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_564_begin_0 = const()[name = tensor("op_564_begin_0"), val = tensor([0, -1, 0])]; - tensor var_564_end_0 = const()[name = tensor("op_564_end_0"), val = tensor([1, 16, 256])]; - tensor var_564_end_mask_0 = const()[name = tensor("op_564_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_59)[name = tensor("transpose_75")]; - tensor var_564 = slice_by_index(begin = var_564_begin_0, end = var_564_end_0, end_mask = var_564_end_mask_0, x = conv_out_7)[name = tensor("op_564")]; - tensor var_567_begin_0 = const()[name = tensor("op_567_begin_0"), val = tensor([0, 4, 0])]; - tensor var_567_end_0 = const()[name = tensor("op_567_end_0"), val = tensor([1, 1, 256])]; - tensor var_567_end_mask_0 = const()[name = tensor("op_567_end_mask_0"), val = tensor([true, true, true])]; - tensor var_567 = slice_by_index(begin = var_567_begin_0, end = var_567_end_0, end_mask = var_567_end_mask_0, x = x_3)[name = tensor("op_567")]; - tensor var_570_begin_0 = const()[name = tensor("op_570_begin_0"), val = tensor([0, 1, 0])]; - tensor var_570_end_0 = const()[name = tensor("op_570_end_0"), val = tensor([1, 16, 256])]; - tensor var_570_end_mask_0 = const()[name = tensor("op_570_end_mask_0"), val = tensor([true, true, true])]; - tensor var_570 = slice_by_index(begin = var_570_begin_0, end = var_570_end_0, end_mask = var_570_end_mask_0, x = input_51)[name = tensor("op_570")]; - tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; - tensor rolled_9 = concat(axis = var_34, interleave = rolled_9_interleave_0, values = (var_570, var_567))[name = tensor("rolled_9")]; - tensor clip_8 = clip(alpha = var_22, beta = var_21, x = clip_7)[name = tensor("clip_8")]; - tensor var_575 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_575")]; - tensor var_576_dtype_0 = const()[name = tensor("op_576_dtype_0"), val = tensor("fp32")]; - tensor var_577 = const()[name = tensor("op_577"), val = tensor([1, 16, 1])]; - tensor var_576 = cast(dtype = var_576_dtype_0, x = var_575)[name = tensor("cast_183")]; - tensor fill_mask_9 = reshape(shape = var_577, x = var_576)[name = tensor("fill_mask_9")]; - tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_567)[name = tensor("x_expanded_9")]; - tensor var_581 = sub(x = var_18, y = fill_mask_9)[name = tensor("op_581")]; - tensor var_582 = mul(x = input_51, y = var_581)[name = tensor("op_582")]; - tensor var_583 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_583")]; - tensor filled_window_9 = add(x = var_582, y = var_583)[name = tensor("filled_window_9")]; - tensor var_585 = greater_equal(x = clip_7, y = var_17)[name = tensor("op_585")]; - tensor var_586_dtype_0 = const()[name = tensor("op_586_dtype_0"), val = tensor("fp32")]; - tensor var_587 = const()[name = tensor("op_587"), val = tensor([1, 1, 1])]; - tensor var_586 = cast(dtype = var_586_dtype_0, x = var_585)[name = tensor("cast_182")]; - tensor use_roll_9 = reshape(shape = var_587, x = var_586)[name = tensor("use_roll_9")]; - tensor var_589 = sub(x = var_18, y = use_roll_9)[name = tensor("op_589")]; - tensor var_590 = mul(x = filled_window_9, y = var_589)[name = tensor("op_590")]; - tensor var_591 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_591")]; - tensor input_61 = add(x = var_590, y = var_591)[name = tensor("input_61")]; - tensor var_593 = const()[name = tensor("op_593"), val = tensor(0x1p+0)]; - tensor var_594 = add(x = clip_7, y = var_593)[name = tensor("op_594")]; - tensor clip_9 = clip(alpha = var_22, beta = var_17, x = var_594)[name = tensor("clip_9")]; - tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; - tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_61)[name = tensor("x_13")]; - tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_27_pad_type_0 = const()[name = tensor("inputs_27_pad_type_0"), val = tensor("valid")]; - tensor inputs_27_strides_0 = const()[name = tensor("inputs_27_strides_0"), val = tensor([1])]; - tensor inputs_27_pad_0 = const()[name = tensor("inputs_27_pad_0"), val = tensor([0, 0])]; - tensor inputs_27_dilations_0 = const()[name = tensor("inputs_27_dilations_0"), val = tensor([1])]; - tensor inputs_27_groups_0 = const()[name = tensor("inputs_27_groups_0"), val = tensor(1)]; - tensor input_63 = transpose(perm = input_63_perm_0, x = x_13)[name = tensor("transpose_74")]; - tensor inputs_27 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_27_dilations_0, groups = inputs_27_groups_0, pad = inputs_27_pad_0, pad_type = inputs_27_pad_type_0, strides = inputs_27_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_27")]; - tensor var_615_split_sizes_0 = const()[name = tensor("op_615_split_sizes_0"), val = tensor([256, 256])]; - tensor var_615_axis_0 = const()[name = tensor("op_615_axis_0"), val = tensor(1)]; - tensor var_615_0, tensor var_615_1 = split(axis = var_615_axis_0, split_sizes = var_615_split_sizes_0, x = inputs_27)[name = tensor("op_615")]; - tensor var_617 = sigmoid(x = var_615_1)[name = tensor("op_617")]; - tensor inputs_29 = mul(x = var_615_0, y = var_617)[name = tensor("inputs_29")]; - tensor outputs_aug_9_pad_type_0 = const()[name = tensor("outputs_aug_9_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_9_pad_0 = const()[name = tensor("outputs_aug_9_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_9_groups_0 = const()[name = tensor("outputs_aug_9_groups_0"), val = tensor(256)]; - tensor outputs_aug_9_strides_0 = const()[name = tensor("outputs_aug_9_strides_0"), val = tensor([1])]; - tensor outputs_aug_9_dilations_0 = const()[name = tensor("outputs_aug_9_dilations_0"), val = tensor([1])]; - tensor outputs_aug_9 = conv(dilations = outputs_aug_9_dilations_0, groups = outputs_aug_9_groups_0, pad = outputs_aug_9_pad_0, pad_type = outputs_aug_9_pad_type_0, strides = outputs_aug_9_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_29)[name = tensor("outputs_aug_9")]; - tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; - tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([1, 256, 16])]; - tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; - tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_9)[name = tensor("input_65")]; - tensor inputs_31 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_65)[name = tensor("inputs_31")]; - tensor input_67 = silu(x = inputs_31)[name = tensor("input_67")]; - tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; - tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; - tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; - tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; - tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; - tensor input_69 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; - tensor conv_out_9_perm_0 = const()[name = tensor("conv_out_9_perm_0"), val = tensor([0, 2, 1])]; - tensor var_648_begin_0 = const()[name = tensor("op_648_begin_0"), val = tensor([0, -1, 0])]; - tensor var_648_end_0 = const()[name = tensor("op_648_end_0"), val = tensor([1, 16, 256])]; - tensor var_648_end_mask_0 = const()[name = tensor("op_648_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_9 = transpose(perm = conv_out_9_perm_0, x = input_69)[name = tensor("transpose_73")]; - tensor var_648 = slice_by_index(begin = var_648_begin_0, end = var_648_end_0, end_mask = var_648_end_mask_0, x = conv_out_9)[name = tensor("op_648")]; - tensor var_651_interleave_0 = const()[name = tensor("op_651_interleave_0"), val = tensor(false)]; - tensor var_651 = concat(axis = var_34, interleave = var_651_interleave_0, values = (var_312, var_396, var_480, var_564, var_648))[name = tensor("op_651")]; - tensor input_71 = add(x = x_3, y = var_651)[name = tensor("input_71")]; - tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; - tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; - tensor inputs_33 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_8")]; - tensor input_75 = silu(x = inputs_33)[name = tensor("input_75")]; - tensor input_79 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_9")]; - tensor var_674 = const()[name = tensor("op_674"), val = tensor(0x1p-1)]; - tensor var_675 = mul(x = input_79, y = var_674)[name = tensor("op_675")]; - tensor input_81 = add(x = var_675, y = input_71)[name = tensor("input_81")]; - tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; - tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_36, gamma = encoder_layer_norm_0_weight, x = input_81)[name = tensor("input_83")]; - tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; - tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; - tensor inputs_35 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_10")]; - tensor input_87 = silu(x = inputs_35)[name = tensor("input_87")]; - tensor input_91 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_11")]; - tensor var_704 = const()[name = tensor("op_704"), val = tensor(0x1p-1)]; - tensor var_705 = mul(x = input_91, y = var_704)[name = tensor("op_705")]; - tensor input_93 = add(x = var_705, y = input_83)[name = tensor("input_93")]; - tensor x_15_axes_0 = const()[name = tensor("x_15_axes_0"), val = tensor([-1])]; - tensor x_15 = layer_norm(axes = x_15_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_36, gamma = encoder_ret_lns_1_weight, x = input_93)[name = tensor("x_15")]; + tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_36, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_3_end_0 = const()[name = tensor("prev_kv_3_end_0"), val = tensor([2, 1, 4, 64, 64])]; tensor prev_kv_3_end_mask_0 = const()[name = tensor("prev_kv_3_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -709,462 +556,308 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_719 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_15)[name = tensor("linear_12")]; - tensor var_720 = const()[name = tensor("op_720"), val = tensor([1, 5, 4, 64])]; - tensor var_721 = reshape(shape = var_720, x = var_719)[name = tensor("op_721")]; + tensor var_504 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_505 = const()[name = tensor("op_505"), val = tensor([1, 5, 4, 64])]; + tensor var_506 = reshape(shape = var_505, x = var_504)[name = tensor("op_506")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_725 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_15)[name = tensor("linear_13")]; - tensor var_726 = const()[name = tensor("op_726"), val = tensor(0x1p-3)]; - tensor var_727 = mul(x = var_725, y = var_726)[name = tensor("op_727")]; - tensor var_728 = const()[name = tensor("op_728"), val = tensor([1, 5, 4, 64])]; - tensor var_729 = reshape(shape = var_728, x = var_727)[name = tensor("op_729")]; + tensor var_510 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_511 = const()[name = tensor("op_511"), val = tensor(0x1p-3)]; + tensor var_512 = mul(x = var_510, y = var_511)[name = tensor("op_512")]; + tensor var_513 = const()[name = tensor("op_513"), val = tensor([1, 5, 4, 64])]; + tensor var_514 = reshape(shape = var_513, x = var_512)[name = tensor("op_514")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_733 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_15)[name = tensor("linear_14")]; - tensor var_734 = const()[name = tensor("op_734"), val = tensor([1, 5, 4, 64])]; - tensor var_735 = reshape(shape = var_734, x = var_733)[name = tensor("op_735")]; + tensor var_518 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_519 = const()[name = tensor("op_519"), val = tensor([1, 5, 4, 64])]; + tensor var_520 = reshape(shape = var_519, x = var_518)[name = tensor("op_520")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_97 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_15)[name = tensor("linear_15")]; + tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; tensor s_t_3 = add(x = prev_scale_3, y = encoder__t_index)[name = tensor("s_t_3")]; tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_729)[name = tensor("transpose_71")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_721)[name = tensor("transpose_72")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_514)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_506)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_745 = const()[name = tensor("op_745"), val = tensor([5, 1])]; - tensor var_746 = reshape(shape = var_745, x = sqrt_s_t_3)[name = tensor("op_746")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_746)[name = tensor("M_3")]; - tensor var_748 = mul(x = qk_3, y = M_3)[name = tensor("op_748")]; + tensor var_530 = const()[name = tensor("op_530"), val = tensor([5, 1])]; + tensor var_531 = reshape(shape = var_530, x = sqrt_s_t_3)[name = tensor("op_531")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_531)[name = tensor("M_3")]; + tensor var_533 = mul(x = qk_3, y = M_3)[name = tensor("op_533")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_735)[name = tensor("transpose_70")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_748, y = v_3)[name = tensor("inner_3")]; - tensor var_750_transpose_x_0 = const()[name = tensor("op_750_transpose_x_0"), val = tensor(false)]; - tensor var_750_transpose_y_0 = const()[name = tensor("op_750_transpose_y_0"), val = tensor(false)]; - tensor var_750 = matmul(transpose_x = var_750_transpose_x_0, transpose_y = var_750_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_750")]; - tensor var_751 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_751")]; - tensor var_752 = const()[name = tensor("op_752"), val = tensor([1, 1, 5, 1])]; - tensor var_753 = reshape(shape = var_752, x = var_751)[name = tensor("op_753")]; - tensor cross_3 = mul(x = var_750, y = var_753)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_520)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_533, y = v_3)[name = tensor("inner_3")]; + tensor var_535_transpose_x_0 = const()[name = tensor("op_535_transpose_x_0"), val = tensor(false)]; + tensor var_535_transpose_y_0 = const()[name = tensor("op_535_transpose_y_0"), val = tensor(false)]; + tensor var_535 = matmul(transpose_x = var_535_transpose_x_0, transpose_y = var_535_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_535")]; + tensor var_536 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_536")]; + tensor var_537 = const()[name = tensor("op_537"), val = tensor([1, 1, 5, 1])]; + tensor var_538 = reshape(shape = var_537, x = var_536)[name = tensor("op_538")]; + tensor cross_3 = mul(x = var_535, y = var_538)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_756 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_756")]; - tensor var_758_transpose_x_1 = const()[name = tensor("op_758_transpose_x_1"), val = tensor(true)]; - tensor var_758_transpose_y_1 = const()[name = tensor("op_758_transpose_y_1"), val = tensor(false)]; - tensor var_758 = matmul(transpose_x = var_758_transpose_x_1, transpose_y = var_758_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_758")]; - tensor new_kv_unnorm_3 = add(x = var_756, y = var_758)[name = tensor("new_kv_unnorm_3")]; - tensor var_760 = const()[name = tensor("op_760"), val = tensor(0x1.4p+2)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_760)[name = tensor("new_scale_3")]; - tensor var_762 = sqrt(x = new_scale_3)[name = tensor("op_762")]; - tensor var_763 = real_div(x = new_kv_unnorm_3, y = var_762)[name = tensor("op_763")]; - tensor var_764_perm_0 = const()[name = tensor("op_764_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_541 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_541")]; + tensor var_543_transpose_x_1 = const()[name = tensor("op_543_transpose_x_1"), val = tensor(true)]; + tensor var_543_transpose_y_1 = const()[name = tensor("op_543_transpose_y_1"), val = tensor(false)]; + tensor var_543 = matmul(transpose_x = var_543_transpose_x_1, transpose_y = var_543_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_543")]; + tensor new_kv_unnorm_3 = add(x = var_541, y = var_543)[name = tensor("new_kv_unnorm_3")]; + tensor var_545 = const()[name = tensor("op_545"), val = tensor(0x1.4p+2)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_545)[name = tensor("new_scale_3")]; + tensor var_547 = sqrt(x = new_scale_3)[name = tensor("op_547")]; + tensor var_548 = real_div(x = new_kv_unnorm_3, y = var_547)[name = tensor("op_548")]; + tensor var_549_perm_0 = const()[name = tensor("op_549_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_764 = transpose(perm = var_764_perm_0, x = out_7)[name = tensor("transpose_69")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_25, x = var_764)[name = tensor("out_9")]; - tensor var_768 = const()[name = tensor("op_768"), val = tensor([1, 5, 256])]; - tensor out_11 = reshape(shape = var_768, x = out_9)[name = tensor("out_11")]; - tensor var_770 = silu(x = input_97)[name = tensor("op_770")]; - tensor input_99 = mul(x = var_770, y = out_11)[name = tensor("input_99")]; - tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_99)[name = tensor("linear_16")]; - tensor x_17 = add(x = input_93, y = ret_out_3)[name = tensor("x_17")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor var_549 = transpose(perm = var_549_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_25, x = var_549)[name = tensor("out_9")]; + tensor var_553 = const()[name = tensor("op_553"), val = tensor([1, 5, 256])]; + tensor out_11 = reshape(shape = var_553, x = out_9)[name = tensor("out_11")]; + tensor var_555 = silu(x = input_57)[name = tensor("op_555")]; + tensor input_59 = mul(x = var_555, y = out_11)[name = tensor("input_59")]; + tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; + tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; + tensor window_13_begin_0 = const()[name = tensor("window_13_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_13_end_0 = const()[name = tensor("window_13_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_13_end_mask_0 = const()[name = tensor("window_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_13_squeeze_mask_0 = const()[name = tensor("window_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_13 = slice_by_index(begin = window_13_begin_0, end = window_13_end_0, end_mask = window_13_end_mask_0, squeeze_mask = window_13_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_13")]; tensor filled_11_begin_0 = const()[name = tensor("filled_11_begin_0"), val = tensor([1, 0])]; tensor filled_11_end_0 = const()[name = tensor("filled_11_end_0"), val = tensor([2, 1])]; tensor filled_11_end_mask_0 = const()[name = tensor("filled_11_end_mask_0"), val = tensor([false, true])]; tensor filled_11_squeeze_mask_0 = const()[name = tensor("filled_11_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_11 = slice_by_index(begin = filled_11_begin_0, end = filled_11_end_0, end_mask = filled_11_end_mask_0, squeeze_mask = filled_11_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_11")]; - tensor var_779_begin_0 = const()[name = tensor("op_779_begin_0"), val = tensor([0, 0, 0])]; - tensor var_779_end_0 = const()[name = tensor("op_779_end_0"), val = tensor([1, 1, 256])]; - tensor var_779_end_mask_0 = const()[name = tensor("op_779_end_mask_0"), val = tensor([true, false, true])]; - tensor var_779 = slice_by_index(begin = var_779_begin_0, end = var_779_end_0, end_mask = var_779_end_mask_0, x = x_17)[name = tensor("op_779")]; - tensor var_782_begin_0 = const()[name = tensor("op_782_begin_0"), val = tensor([0, 1, 0])]; - tensor var_782_end_0 = const()[name = tensor("op_782_end_0"), val = tensor([1, 16, 256])]; - tensor var_782_end_mask_0 = const()[name = tensor("op_782_end_mask_0"), val = tensor([true, true, true])]; - tensor var_782 = slice_by_index(begin = var_782_begin_0, end = var_782_end_0, end_mask = var_782_end_mask_0, x = window_3)[name = tensor("op_782")]; + tensor var_564_begin_0 = const()[name = tensor("op_564_begin_0"), val = tensor([0, 0, 0])]; + tensor var_564_end_0 = const()[name = tensor("op_564_end_0"), val = tensor([1, 1, 256])]; + tensor var_564_end_mask_0 = const()[name = tensor("op_564_end_mask_0"), val = tensor([true, false, true])]; + tensor var_564 = slice_by_index(begin = var_564_begin_0, end = var_564_end_0, end_mask = var_564_end_mask_0, x = x_9)[name = tensor("op_564")]; + tensor var_567_begin_0 = const()[name = tensor("op_567_begin_0"), val = tensor([0, 1, 0])]; + tensor var_567_end_0 = const()[name = tensor("op_567_end_0"), val = tensor([1, 16, 256])]; + tensor var_567_end_mask_0 = const()[name = tensor("op_567_end_mask_0"), val = tensor([true, true, true])]; + tensor var_567 = slice_by_index(begin = var_567_begin_0, end = var_567_end_0, end_mask = var_567_end_mask_0, x = window_13)[name = tensor("op_567")]; tensor rolled_11_interleave_0 = const()[name = tensor("rolled_11_interleave_0"), val = tensor(false)]; - tensor rolled_11 = concat(axis = var_34, interleave = rolled_11_interleave_0, values = (var_782, var_779))[name = tensor("rolled_11")]; + tensor rolled_11 = concat(axis = var_34, interleave = rolled_11_interleave_0, values = (var_567, var_564))[name = tensor("rolled_11")]; tensor clip_10 = clip(alpha = var_22, beta = var_21, x = filled_11)[name = tensor("clip_10")]; - tensor var_787 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_787")]; - tensor var_788_dtype_0 = const()[name = tensor("op_788_dtype_0"), val = tensor("fp32")]; - tensor var_789 = const()[name = tensor("op_789"), val = tensor([1, 16, 1])]; - tensor var_788 = cast(dtype = var_788_dtype_0, x = var_787)[name = tensor("cast_181")]; - tensor fill_mask_11 = reshape(shape = var_789, x = var_788)[name = tensor("fill_mask_11")]; + tensor var_572 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_572")]; + tensor var_573_dtype_0 = const()[name = tensor("op_573_dtype_0"), val = tensor("fp32")]; + tensor var_574 = const()[name = tensor("op_574"), val = tensor([1, 16, 1])]; + tensor var_573 = cast(dtype = var_573_dtype_0, x = var_572)[name = tensor("cast_181")]; + tensor fill_mask_11 = reshape(shape = var_574, x = var_573)[name = tensor("fill_mask_11")]; tensor x_expanded_11_reps_0 = const()[name = tensor("x_expanded_11_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_779)[name = tensor("x_expanded_11")]; - tensor var_793 = sub(x = var_18, y = fill_mask_11)[name = tensor("op_793")]; - tensor var_794 = mul(x = window_3, y = var_793)[name = tensor("op_794")]; - tensor var_795 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_795")]; - tensor filled_window_11 = add(x = var_794, y = var_795)[name = tensor("filled_window_11")]; - tensor var_797 = greater_equal(x = filled_11, y = var_17)[name = tensor("op_797")]; - tensor var_798_dtype_0 = const()[name = tensor("op_798_dtype_0"), val = tensor("fp32")]; - tensor var_799 = const()[name = tensor("op_799"), val = tensor([1, 1, 1])]; - tensor var_798 = cast(dtype = var_798_dtype_0, x = var_797)[name = tensor("cast_180")]; - tensor use_roll_11 = reshape(shape = var_799, x = var_798)[name = tensor("use_roll_11")]; - tensor var_801 = sub(x = var_18, y = use_roll_11)[name = tensor("op_801")]; - tensor var_802 = mul(x = filled_window_11, y = var_801)[name = tensor("op_802")]; - tensor var_803 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_803")]; - tensor input_101 = add(x = var_802, y = var_803)[name = tensor("input_101")]; - tensor var_805 = const()[name = tensor("op_805"), val = tensor(0x1p+0)]; - tensor var_806 = add(x = filled_11, y = var_805)[name = tensor("op_806")]; - tensor clip_11 = clip(alpha = var_22, beta = var_17, x = var_806)[name = tensor("clip_11")]; - tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; - tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_101)[name = tensor("x_19")]; - tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_37_pad_type_0 = const()[name = tensor("inputs_37_pad_type_0"), val = tensor("valid")]; - tensor inputs_37_strides_0 = const()[name = tensor("inputs_37_strides_0"), val = tensor([1])]; - tensor inputs_37_pad_0 = const()[name = tensor("inputs_37_pad_0"), val = tensor([0, 0])]; - tensor inputs_37_dilations_0 = const()[name = tensor("inputs_37_dilations_0"), val = tensor([1])]; - tensor inputs_37_groups_0 = const()[name = tensor("inputs_37_groups_0"), val = tensor(1)]; - tensor input_103 = transpose(perm = input_103_perm_0, x = x_19)[name = tensor("transpose_68")]; - tensor inputs_37 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_37_dilations_0, groups = inputs_37_groups_0, pad = inputs_37_pad_0, pad_type = inputs_37_pad_type_0, strides = inputs_37_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_37")]; - tensor var_827_split_sizes_0 = const()[name = tensor("op_827_split_sizes_0"), val = tensor([256, 256])]; - tensor var_827_axis_0 = const()[name = tensor("op_827_axis_0"), val = tensor(1)]; - tensor var_827_0, tensor var_827_1 = split(axis = var_827_axis_0, split_sizes = var_827_split_sizes_0, x = inputs_37)[name = tensor("op_827")]; - tensor var_829 = sigmoid(x = var_827_1)[name = tensor("op_829")]; - tensor inputs_39 = mul(x = var_827_0, y = var_829)[name = tensor("inputs_39")]; - tensor outputs_aug_11_pad_type_0 = const()[name = tensor("outputs_aug_11_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_11_pad_0 = const()[name = tensor("outputs_aug_11_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_11_groups_0 = const()[name = tensor("outputs_aug_11_groups_0"), val = tensor(256)]; - tensor outputs_aug_11_strides_0 = const()[name = tensor("outputs_aug_11_strides_0"), val = tensor([1])]; - tensor outputs_aug_11_dilations_0 = const()[name = tensor("outputs_aug_11_dilations_0"), val = tensor([1])]; - tensor outputs_aug_11 = conv(dilations = outputs_aug_11_dilations_0, groups = outputs_aug_11_groups_0, pad = outputs_aug_11_pad_0, pad_type = outputs_aug_11_pad_type_0, strides = outputs_aug_11_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_39)[name = tensor("outputs_aug_11")]; - tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; - tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([1, 256, 16])]; - tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; - tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_11)[name = tensor("input_105")]; - tensor inputs_41 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_105)[name = tensor("inputs_41")]; - tensor input_107 = silu(x = inputs_41)[name = tensor("input_107")]; - tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; - tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; - tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; - tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; - tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; - tensor input_109 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; - tensor conv_out_11_perm_0 = const()[name = tensor("conv_out_11_perm_0"), val = tensor([0, 2, 1])]; - tensor var_860_begin_0 = const()[name = tensor("op_860_begin_0"), val = tensor([0, -1, 0])]; - tensor var_860_end_0 = const()[name = tensor("op_860_end_0"), val = tensor([1, 16, 256])]; - tensor var_860_end_mask_0 = const()[name = tensor("op_860_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_11 = transpose(perm = conv_out_11_perm_0, x = input_109)[name = tensor("transpose_67")]; - tensor var_860 = slice_by_index(begin = var_860_begin_0, end = var_860_end_0, end_mask = var_860_end_mask_0, x = conv_out_11)[name = tensor("op_860")]; - tensor var_863_begin_0 = const()[name = tensor("op_863_begin_0"), val = tensor([0, 1, 0])]; - tensor var_863_end_0 = const()[name = tensor("op_863_end_0"), val = tensor([1, 2, 256])]; - tensor var_863_end_mask_0 = const()[name = tensor("op_863_end_mask_0"), val = tensor([true, false, true])]; - tensor var_863 = slice_by_index(begin = var_863_begin_0, end = var_863_end_0, end_mask = var_863_end_mask_0, x = x_17)[name = tensor("op_863")]; - tensor var_866_begin_0 = const()[name = tensor("op_866_begin_0"), val = tensor([0, 1, 0])]; - tensor var_866_end_0 = const()[name = tensor("op_866_end_0"), val = tensor([1, 16, 256])]; - tensor var_866_end_mask_0 = const()[name = tensor("op_866_end_mask_0"), val = tensor([true, true, true])]; - tensor var_866 = slice_by_index(begin = var_866_begin_0, end = var_866_end_0, end_mask = var_866_end_mask_0, x = input_101)[name = tensor("op_866")]; + tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_564)[name = tensor("x_expanded_11")]; + tensor var_578 = sub(x = var_18, y = fill_mask_11)[name = tensor("op_578")]; + tensor var_579 = mul(x = window_13, y = var_578)[name = tensor("op_579")]; + tensor var_580 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_580")]; + tensor filled_window_11 = add(x = var_579, y = var_580)[name = tensor("filled_window_11")]; + tensor var_582 = greater_equal(x = filled_11, y = var_17)[name = tensor("op_582")]; + tensor var_583_dtype_0 = const()[name = tensor("op_583_dtype_0"), val = tensor("fp32")]; + tensor var_584 = const()[name = tensor("op_584"), val = tensor([1, 1, 1])]; + tensor var_583 = cast(dtype = var_583_dtype_0, x = var_582)[name = tensor("cast_180")]; + tensor use_roll_11 = reshape(shape = var_584, x = var_583)[name = tensor("use_roll_11")]; + tensor var_586 = sub(x = var_18, y = use_roll_11)[name = tensor("op_586")]; + tensor var_587 = mul(x = filled_window_11, y = var_586)[name = tensor("op_587")]; + tensor var_588 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_588")]; + tensor window_15 = add(x = var_587, y = var_588)[name = tensor("window_15")]; + tensor var_590 = const()[name = tensor("op_590"), val = tensor(0x1p+0)]; + tensor var_591 = add(x = filled_11, y = var_590)[name = tensor("op_591")]; + tensor clip_11 = clip(alpha = var_22, beta = var_17, x = var_591)[name = tensor("clip_11")]; + tensor var_594_begin_0 = const()[name = tensor("op_594_begin_0"), val = tensor([0, 1, 0])]; + tensor var_594_end_0 = const()[name = tensor("op_594_end_0"), val = tensor([1, 2, 256])]; + tensor var_594_end_mask_0 = const()[name = tensor("op_594_end_mask_0"), val = tensor([true, false, true])]; + tensor var_594 = slice_by_index(begin = var_594_begin_0, end = var_594_end_0, end_mask = var_594_end_mask_0, x = x_9)[name = tensor("op_594")]; + tensor var_597_begin_0 = const()[name = tensor("op_597_begin_0"), val = tensor([0, 1, 0])]; + tensor var_597_end_0 = const()[name = tensor("op_597_end_0"), val = tensor([1, 16, 256])]; + tensor var_597_end_mask_0 = const()[name = tensor("op_597_end_mask_0"), val = tensor([true, true, true])]; + tensor var_597 = slice_by_index(begin = var_597_begin_0, end = var_597_end_0, end_mask = var_597_end_mask_0, x = window_15)[name = tensor("op_597")]; tensor rolled_13_interleave_0 = const()[name = tensor("rolled_13_interleave_0"), val = tensor(false)]; - tensor rolled_13 = concat(axis = var_34, interleave = rolled_13_interleave_0, values = (var_866, var_863))[name = tensor("rolled_13")]; + tensor rolled_13 = concat(axis = var_34, interleave = rolled_13_interleave_0, values = (var_597, var_594))[name = tensor("rolled_13")]; tensor clip_12 = clip(alpha = var_22, beta = var_21, x = clip_11)[name = tensor("clip_12")]; - tensor var_871 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_871")]; - tensor var_872_dtype_0 = const()[name = tensor("op_872_dtype_0"), val = tensor("fp32")]; - tensor var_873 = const()[name = tensor("op_873"), val = tensor([1, 16, 1])]; - tensor var_872 = cast(dtype = var_872_dtype_0, x = var_871)[name = tensor("cast_179")]; - tensor fill_mask_13 = reshape(shape = var_873, x = var_872)[name = tensor("fill_mask_13")]; + tensor var_602 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_602")]; + tensor var_603_dtype_0 = const()[name = tensor("op_603_dtype_0"), val = tensor("fp32")]; + tensor var_604 = const()[name = tensor("op_604"), val = tensor([1, 16, 1])]; + tensor var_603 = cast(dtype = var_603_dtype_0, x = var_602)[name = tensor("cast_179")]; + tensor fill_mask_13 = reshape(shape = var_604, x = var_603)[name = tensor("fill_mask_13")]; tensor x_expanded_13_reps_0 = const()[name = tensor("x_expanded_13_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_863)[name = tensor("x_expanded_13")]; - tensor var_877 = sub(x = var_18, y = fill_mask_13)[name = tensor("op_877")]; - tensor var_878 = mul(x = input_101, y = var_877)[name = tensor("op_878")]; - tensor var_879 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_879")]; - tensor filled_window_13 = add(x = var_878, y = var_879)[name = tensor("filled_window_13")]; - tensor var_881 = greater_equal(x = clip_11, y = var_17)[name = tensor("op_881")]; - tensor var_882_dtype_0 = const()[name = tensor("op_882_dtype_0"), val = tensor("fp32")]; - tensor var_883 = const()[name = tensor("op_883"), val = tensor([1, 1, 1])]; - tensor var_882 = cast(dtype = var_882_dtype_0, x = var_881)[name = tensor("cast_178")]; - tensor use_roll_13 = reshape(shape = var_883, x = var_882)[name = tensor("use_roll_13")]; - tensor var_885 = sub(x = var_18, y = use_roll_13)[name = tensor("op_885")]; - tensor var_886 = mul(x = filled_window_13, y = var_885)[name = tensor("op_886")]; - tensor var_887 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_887")]; - tensor input_111 = add(x = var_886, y = var_887)[name = tensor("input_111")]; - tensor var_889 = const()[name = tensor("op_889"), val = tensor(0x1p+0)]; - tensor var_890 = add(x = clip_11, y = var_889)[name = tensor("op_890")]; - tensor clip_13 = clip(alpha = var_22, beta = var_17, x = var_890)[name = tensor("clip_13")]; - tensor x_21_axes_0 = const()[name = tensor("x_21_axes_0"), val = tensor([-1])]; - tensor x_21 = layer_norm(axes = x_21_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_111)[name = tensor("x_21")]; - tensor input_113_perm_0 = const()[name = tensor("input_113_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_43_pad_type_0 = const()[name = tensor("inputs_43_pad_type_0"), val = tensor("valid")]; - tensor inputs_43_strides_0 = const()[name = tensor("inputs_43_strides_0"), val = tensor([1])]; - tensor inputs_43_pad_0 = const()[name = tensor("inputs_43_pad_0"), val = tensor([0, 0])]; - tensor inputs_43_dilations_0 = const()[name = tensor("inputs_43_dilations_0"), val = tensor([1])]; - tensor inputs_43_groups_0 = const()[name = tensor("inputs_43_groups_0"), val = tensor(1)]; - tensor input_113 = transpose(perm = input_113_perm_0, x = x_21)[name = tensor("transpose_66")]; - tensor inputs_43 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_43_dilations_0, groups = inputs_43_groups_0, pad = inputs_43_pad_0, pad_type = inputs_43_pad_type_0, strides = inputs_43_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_113)[name = tensor("inputs_43")]; - tensor var_911_split_sizes_0 = const()[name = tensor("op_911_split_sizes_0"), val = tensor([256, 256])]; - tensor var_911_axis_0 = const()[name = tensor("op_911_axis_0"), val = tensor(1)]; - tensor var_911_0, tensor var_911_1 = split(axis = var_911_axis_0, split_sizes = var_911_split_sizes_0, x = inputs_43)[name = tensor("op_911")]; - tensor var_913 = sigmoid(x = var_911_1)[name = tensor("op_913")]; - tensor inputs_45 = mul(x = var_911_0, y = var_913)[name = tensor("inputs_45")]; - tensor outputs_aug_13_pad_type_0 = const()[name = tensor("outputs_aug_13_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_13_pad_0 = const()[name = tensor("outputs_aug_13_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_13_groups_0 = const()[name = tensor("outputs_aug_13_groups_0"), val = tensor(256)]; - tensor outputs_aug_13_strides_0 = const()[name = tensor("outputs_aug_13_strides_0"), val = tensor([1])]; - tensor outputs_aug_13_dilations_0 = const()[name = tensor("outputs_aug_13_dilations_0"), val = tensor([1])]; - tensor outputs_aug_13 = conv(dilations = outputs_aug_13_dilations_0, groups = outputs_aug_13_groups_0, pad = outputs_aug_13_pad_0, pad_type = outputs_aug_13_pad_type_0, strides = outputs_aug_13_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_45)[name = tensor("outputs_aug_13")]; - tensor input_115_begin_0 = const()[name = tensor("input_115_begin_0"), val = tensor([0, 0, 0])]; - tensor input_115_end_0 = const()[name = tensor("input_115_end_0"), val = tensor([1, 256, 16])]; - tensor input_115_end_mask_0 = const()[name = tensor("input_115_end_mask_0"), val = tensor([true, true, false])]; - tensor input_115 = slice_by_index(begin = input_115_begin_0, end = input_115_end_0, end_mask = input_115_end_mask_0, x = outputs_aug_13)[name = tensor("input_115")]; - tensor inputs_47 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_115)[name = tensor("inputs_47")]; - tensor input_117 = silu(x = inputs_47)[name = tensor("input_117")]; - tensor input_119_pad_type_0 = const()[name = tensor("input_119_pad_type_0"), val = tensor("valid")]; - tensor input_119_strides_0 = const()[name = tensor("input_119_strides_0"), val = tensor([1])]; - tensor input_119_pad_0 = const()[name = tensor("input_119_pad_0"), val = tensor([0, 0])]; - tensor input_119_dilations_0 = const()[name = tensor("input_119_dilations_0"), val = tensor([1])]; - tensor input_119_groups_0 = const()[name = tensor("input_119_groups_0"), val = tensor(1)]; - tensor input_119 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_119_dilations_0, groups = input_119_groups_0, pad = input_119_pad_0, pad_type = input_119_pad_type_0, strides = input_119_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_117)[name = tensor("input_119")]; - tensor conv_out_13_perm_0 = const()[name = tensor("conv_out_13_perm_0"), val = tensor([0, 2, 1])]; - tensor var_944_begin_0 = const()[name = tensor("op_944_begin_0"), val = tensor([0, -1, 0])]; - tensor var_944_end_0 = const()[name = tensor("op_944_end_0"), val = tensor([1, 16, 256])]; - tensor var_944_end_mask_0 = const()[name = tensor("op_944_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_13 = transpose(perm = conv_out_13_perm_0, x = input_119)[name = tensor("transpose_65")]; - tensor var_944 = slice_by_index(begin = var_944_begin_0, end = var_944_end_0, end_mask = var_944_end_mask_0, x = conv_out_13)[name = tensor("op_944")]; - tensor var_947_begin_0 = const()[name = tensor("op_947_begin_0"), val = tensor([0, 2, 0])]; - tensor var_947_end_0 = const()[name = tensor("op_947_end_0"), val = tensor([1, 3, 256])]; - tensor var_947_end_mask_0 = const()[name = tensor("op_947_end_mask_0"), val = tensor([true, false, true])]; - tensor var_947 = slice_by_index(begin = var_947_begin_0, end = var_947_end_0, end_mask = var_947_end_mask_0, x = x_17)[name = tensor("op_947")]; - tensor var_950_begin_0 = const()[name = tensor("op_950_begin_0"), val = tensor([0, 1, 0])]; - tensor var_950_end_0 = const()[name = tensor("op_950_end_0"), val = tensor([1, 16, 256])]; - tensor var_950_end_mask_0 = const()[name = tensor("op_950_end_mask_0"), val = tensor([true, true, true])]; - tensor var_950 = slice_by_index(begin = var_950_begin_0, end = var_950_end_0, end_mask = var_950_end_mask_0, x = input_111)[name = tensor("op_950")]; + tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_594)[name = tensor("x_expanded_13")]; + tensor var_608 = sub(x = var_18, y = fill_mask_13)[name = tensor("op_608")]; + tensor var_609 = mul(x = window_15, y = var_608)[name = tensor("op_609")]; + tensor var_610 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_610")]; + tensor filled_window_13 = add(x = var_609, y = var_610)[name = tensor("filled_window_13")]; + tensor var_612 = greater_equal(x = clip_11, y = var_17)[name = tensor("op_612")]; + tensor var_613_dtype_0 = const()[name = tensor("op_613_dtype_0"), val = tensor("fp32")]; + tensor var_614 = const()[name = tensor("op_614"), val = tensor([1, 1, 1])]; + tensor var_613 = cast(dtype = var_613_dtype_0, x = var_612)[name = tensor("cast_178")]; + tensor use_roll_13 = reshape(shape = var_614, x = var_613)[name = tensor("use_roll_13")]; + tensor var_616 = sub(x = var_18, y = use_roll_13)[name = tensor("op_616")]; + tensor var_617 = mul(x = filled_window_13, y = var_616)[name = tensor("op_617")]; + tensor var_618 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_618")]; + tensor window_17 = add(x = var_617, y = var_618)[name = tensor("window_17")]; + tensor var_620 = const()[name = tensor("op_620"), val = tensor(0x1p+0)]; + tensor var_621 = add(x = clip_11, y = var_620)[name = tensor("op_621")]; + tensor clip_13 = clip(alpha = var_22, beta = var_17, x = var_621)[name = tensor("clip_13")]; + tensor var_624_begin_0 = const()[name = tensor("op_624_begin_0"), val = tensor([0, 2, 0])]; + tensor var_624_end_0 = const()[name = tensor("op_624_end_0"), val = tensor([1, 3, 256])]; + tensor var_624_end_mask_0 = const()[name = tensor("op_624_end_mask_0"), val = tensor([true, false, true])]; + tensor var_624 = slice_by_index(begin = var_624_begin_0, end = var_624_end_0, end_mask = var_624_end_mask_0, x = x_9)[name = tensor("op_624")]; + tensor var_627_begin_0 = const()[name = tensor("op_627_begin_0"), val = tensor([0, 1, 0])]; + tensor var_627_end_0 = const()[name = tensor("op_627_end_0"), val = tensor([1, 16, 256])]; + tensor var_627_end_mask_0 = const()[name = tensor("op_627_end_mask_0"), val = tensor([true, true, true])]; + tensor var_627 = slice_by_index(begin = var_627_begin_0, end = var_627_end_0, end_mask = var_627_end_mask_0, x = window_17)[name = tensor("op_627")]; tensor rolled_15_interleave_0 = const()[name = tensor("rolled_15_interleave_0"), val = tensor(false)]; - tensor rolled_15 = concat(axis = var_34, interleave = rolled_15_interleave_0, values = (var_950, var_947))[name = tensor("rolled_15")]; + tensor rolled_15 = concat(axis = var_34, interleave = rolled_15_interleave_0, values = (var_627, var_624))[name = tensor("rolled_15")]; tensor clip_14 = clip(alpha = var_22, beta = var_21, x = clip_13)[name = tensor("clip_14")]; - tensor var_955 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_955")]; - tensor var_956_dtype_0 = const()[name = tensor("op_956_dtype_0"), val = tensor("fp32")]; - tensor var_957 = const()[name = tensor("op_957"), val = tensor([1, 16, 1])]; - tensor var_956 = cast(dtype = var_956_dtype_0, x = var_955)[name = tensor("cast_177")]; - tensor fill_mask_15 = reshape(shape = var_957, x = var_956)[name = tensor("fill_mask_15")]; + tensor var_632 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_632")]; + tensor var_633_dtype_0 = const()[name = tensor("op_633_dtype_0"), val = tensor("fp32")]; + tensor var_634 = const()[name = tensor("op_634"), val = tensor([1, 16, 1])]; + tensor var_633 = cast(dtype = var_633_dtype_0, x = var_632)[name = tensor("cast_177")]; + tensor fill_mask_15 = reshape(shape = var_634, x = var_633)[name = tensor("fill_mask_15")]; tensor x_expanded_15_reps_0 = const()[name = tensor("x_expanded_15_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_947)[name = tensor("x_expanded_15")]; - tensor var_961 = sub(x = var_18, y = fill_mask_15)[name = tensor("op_961")]; - tensor var_962 = mul(x = input_111, y = var_961)[name = tensor("op_962")]; - tensor var_963 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_963")]; - tensor filled_window_15 = add(x = var_962, y = var_963)[name = tensor("filled_window_15")]; - tensor var_965 = greater_equal(x = clip_13, y = var_17)[name = tensor("op_965")]; - tensor var_966_dtype_0 = const()[name = tensor("op_966_dtype_0"), val = tensor("fp32")]; - tensor var_967 = const()[name = tensor("op_967"), val = tensor([1, 1, 1])]; - tensor var_966 = cast(dtype = var_966_dtype_0, x = var_965)[name = tensor("cast_176")]; - tensor use_roll_15 = reshape(shape = var_967, x = var_966)[name = tensor("use_roll_15")]; - tensor var_969 = sub(x = var_18, y = use_roll_15)[name = tensor("op_969")]; - tensor var_970 = mul(x = filled_window_15, y = var_969)[name = tensor("op_970")]; - tensor var_971 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_971")]; - tensor input_121 = add(x = var_970, y = var_971)[name = tensor("input_121")]; - tensor var_973 = const()[name = tensor("op_973"), val = tensor(0x1p+0)]; - tensor var_974 = add(x = clip_13, y = var_973)[name = tensor("op_974")]; - tensor clip_15 = clip(alpha = var_22, beta = var_17, x = var_974)[name = tensor("clip_15")]; - tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; - tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_121)[name = tensor("x_23")]; - tensor input_123_perm_0 = const()[name = tensor("input_123_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_49_pad_type_0 = const()[name = tensor("inputs_49_pad_type_0"), val = tensor("valid")]; - tensor inputs_49_strides_0 = const()[name = tensor("inputs_49_strides_0"), val = tensor([1])]; - tensor inputs_49_pad_0 = const()[name = tensor("inputs_49_pad_0"), val = tensor([0, 0])]; - tensor inputs_49_dilations_0 = const()[name = tensor("inputs_49_dilations_0"), val = tensor([1])]; - tensor inputs_49_groups_0 = const()[name = tensor("inputs_49_groups_0"), val = tensor(1)]; - tensor input_123 = transpose(perm = input_123_perm_0, x = x_23)[name = tensor("transpose_64")]; - tensor inputs_49 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_49_dilations_0, groups = inputs_49_groups_0, pad = inputs_49_pad_0, pad_type = inputs_49_pad_type_0, strides = inputs_49_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_123)[name = tensor("inputs_49")]; - tensor var_995_split_sizes_0 = const()[name = tensor("op_995_split_sizes_0"), val = tensor([256, 256])]; - tensor var_995_axis_0 = const()[name = tensor("op_995_axis_0"), val = tensor(1)]; - tensor var_995_0, tensor var_995_1 = split(axis = var_995_axis_0, split_sizes = var_995_split_sizes_0, x = inputs_49)[name = tensor("op_995")]; - tensor var_997 = sigmoid(x = var_995_1)[name = tensor("op_997")]; - tensor inputs_51 = mul(x = var_995_0, y = var_997)[name = tensor("inputs_51")]; - tensor outputs_aug_15_pad_type_0 = const()[name = tensor("outputs_aug_15_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_15_pad_0 = const()[name = tensor("outputs_aug_15_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_15_groups_0 = const()[name = tensor("outputs_aug_15_groups_0"), val = tensor(256)]; - tensor outputs_aug_15_strides_0 = const()[name = tensor("outputs_aug_15_strides_0"), val = tensor([1])]; - tensor outputs_aug_15_dilations_0 = const()[name = tensor("outputs_aug_15_dilations_0"), val = tensor([1])]; - tensor outputs_aug_15 = conv(dilations = outputs_aug_15_dilations_0, groups = outputs_aug_15_groups_0, pad = outputs_aug_15_pad_0, pad_type = outputs_aug_15_pad_type_0, strides = outputs_aug_15_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_51)[name = tensor("outputs_aug_15")]; - tensor input_125_begin_0 = const()[name = tensor("input_125_begin_0"), val = tensor([0, 0, 0])]; - tensor input_125_end_0 = const()[name = tensor("input_125_end_0"), val = tensor([1, 256, 16])]; - tensor input_125_end_mask_0 = const()[name = tensor("input_125_end_mask_0"), val = tensor([true, true, false])]; - tensor input_125 = slice_by_index(begin = input_125_begin_0, end = input_125_end_0, end_mask = input_125_end_mask_0, x = outputs_aug_15)[name = tensor("input_125")]; - tensor inputs_53 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_125)[name = tensor("inputs_53")]; - tensor input_127 = silu(x = inputs_53)[name = tensor("input_127")]; - tensor input_129_pad_type_0 = const()[name = tensor("input_129_pad_type_0"), val = tensor("valid")]; - tensor input_129_strides_0 = const()[name = tensor("input_129_strides_0"), val = tensor([1])]; - tensor input_129_pad_0 = const()[name = tensor("input_129_pad_0"), val = tensor([0, 0])]; - tensor input_129_dilations_0 = const()[name = tensor("input_129_dilations_0"), val = tensor([1])]; - tensor input_129_groups_0 = const()[name = tensor("input_129_groups_0"), val = tensor(1)]; - tensor input_129 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_129_dilations_0, groups = input_129_groups_0, pad = input_129_pad_0, pad_type = input_129_pad_type_0, strides = input_129_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_127)[name = tensor("input_129")]; - tensor conv_out_15_perm_0 = const()[name = tensor("conv_out_15_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1028_begin_0 = const()[name = tensor("op_1028_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1028_end_0 = const()[name = tensor("op_1028_end_0"), val = tensor([1, 16, 256])]; - tensor var_1028_end_mask_0 = const()[name = tensor("op_1028_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_15 = transpose(perm = conv_out_15_perm_0, x = input_129)[name = tensor("transpose_63")]; - tensor var_1028 = slice_by_index(begin = var_1028_begin_0, end = var_1028_end_0, end_mask = var_1028_end_mask_0, x = conv_out_15)[name = tensor("op_1028")]; - tensor var_1031_begin_0 = const()[name = tensor("op_1031_begin_0"), val = tensor([0, 3, 0])]; - tensor var_1031_end_0 = const()[name = tensor("op_1031_end_0"), val = tensor([1, 4, 256])]; - tensor var_1031_end_mask_0 = const()[name = tensor("op_1031_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1031 = slice_by_index(begin = var_1031_begin_0, end = var_1031_end_0, end_mask = var_1031_end_mask_0, x = x_17)[name = tensor("op_1031")]; - tensor var_1034_begin_0 = const()[name = tensor("op_1034_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1034_end_0 = const()[name = tensor("op_1034_end_0"), val = tensor([1, 16, 256])]; - tensor var_1034_end_mask_0 = const()[name = tensor("op_1034_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1034 = slice_by_index(begin = var_1034_begin_0, end = var_1034_end_0, end_mask = var_1034_end_mask_0, x = input_121)[name = tensor("op_1034")]; + tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_624)[name = tensor("x_expanded_15")]; + tensor var_638 = sub(x = var_18, y = fill_mask_15)[name = tensor("op_638")]; + tensor var_639 = mul(x = window_17, y = var_638)[name = tensor("op_639")]; + tensor var_640 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_640")]; + tensor filled_window_15 = add(x = var_639, y = var_640)[name = tensor("filled_window_15")]; + tensor var_642 = greater_equal(x = clip_13, y = var_17)[name = tensor("op_642")]; + tensor var_643_dtype_0 = const()[name = tensor("op_643_dtype_0"), val = tensor("fp32")]; + tensor var_644 = const()[name = tensor("op_644"), val = tensor([1, 1, 1])]; + tensor var_643 = cast(dtype = var_643_dtype_0, x = var_642)[name = tensor("cast_176")]; + tensor use_roll_15 = reshape(shape = var_644, x = var_643)[name = tensor("use_roll_15")]; + tensor var_646 = sub(x = var_18, y = use_roll_15)[name = tensor("op_646")]; + tensor var_647 = mul(x = filled_window_15, y = var_646)[name = tensor("op_647")]; + tensor var_648 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_648")]; + tensor window_19 = add(x = var_647, y = var_648)[name = tensor("window_19")]; + tensor var_650 = const()[name = tensor("op_650"), val = tensor(0x1p+0)]; + tensor var_651 = add(x = clip_13, y = var_650)[name = tensor("op_651")]; + tensor clip_15 = clip(alpha = var_22, beta = var_17, x = var_651)[name = tensor("clip_15")]; + tensor var_654_begin_0 = const()[name = tensor("op_654_begin_0"), val = tensor([0, 3, 0])]; + tensor var_654_end_0 = const()[name = tensor("op_654_end_0"), val = tensor([1, 4, 256])]; + tensor var_654_end_mask_0 = const()[name = tensor("op_654_end_mask_0"), val = tensor([true, false, true])]; + tensor var_654 = slice_by_index(begin = var_654_begin_0, end = var_654_end_0, end_mask = var_654_end_mask_0, x = x_9)[name = tensor("op_654")]; + tensor var_657_begin_0 = const()[name = tensor("op_657_begin_0"), val = tensor([0, 1, 0])]; + tensor var_657_end_0 = const()[name = tensor("op_657_end_0"), val = tensor([1, 16, 256])]; + tensor var_657_end_mask_0 = const()[name = tensor("op_657_end_mask_0"), val = tensor([true, true, true])]; + tensor var_657 = slice_by_index(begin = var_657_begin_0, end = var_657_end_0, end_mask = var_657_end_mask_0, x = window_19)[name = tensor("op_657")]; tensor rolled_17_interleave_0 = const()[name = tensor("rolled_17_interleave_0"), val = tensor(false)]; - tensor rolled_17 = concat(axis = var_34, interleave = rolled_17_interleave_0, values = (var_1034, var_1031))[name = tensor("rolled_17")]; + tensor rolled_17 = concat(axis = var_34, interleave = rolled_17_interleave_0, values = (var_657, var_654))[name = tensor("rolled_17")]; tensor clip_16 = clip(alpha = var_22, beta = var_21, x = clip_15)[name = tensor("clip_16")]; - tensor var_1039 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_1039")]; - tensor var_1040_dtype_0 = const()[name = tensor("op_1040_dtype_0"), val = tensor("fp32")]; - tensor var_1041 = const()[name = tensor("op_1041"), val = tensor([1, 16, 1])]; - tensor var_1040 = cast(dtype = var_1040_dtype_0, x = var_1039)[name = tensor("cast_175")]; - tensor fill_mask_17 = reshape(shape = var_1041, x = var_1040)[name = tensor("fill_mask_17")]; + tensor var_662 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_662")]; + tensor var_663_dtype_0 = const()[name = tensor("op_663_dtype_0"), val = tensor("fp32")]; + tensor var_664 = const()[name = tensor("op_664"), val = tensor([1, 16, 1])]; + tensor var_663 = cast(dtype = var_663_dtype_0, x = var_662)[name = tensor("cast_175")]; + tensor fill_mask_17 = reshape(shape = var_664, x = var_663)[name = tensor("fill_mask_17")]; tensor x_expanded_17_reps_0 = const()[name = tensor("x_expanded_17_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_1031)[name = tensor("x_expanded_17")]; - tensor var_1045 = sub(x = var_18, y = fill_mask_17)[name = tensor("op_1045")]; - tensor var_1046 = mul(x = input_121, y = var_1045)[name = tensor("op_1046")]; - tensor var_1047 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_1047")]; - tensor filled_window_17 = add(x = var_1046, y = var_1047)[name = tensor("filled_window_17")]; - tensor var_1049 = greater_equal(x = clip_15, y = var_17)[name = tensor("op_1049")]; - tensor var_1050_dtype_0 = const()[name = tensor("op_1050_dtype_0"), val = tensor("fp32")]; - tensor var_1051 = const()[name = tensor("op_1051"), val = tensor([1, 1, 1])]; - tensor var_1050 = cast(dtype = var_1050_dtype_0, x = var_1049)[name = tensor("cast_174")]; - tensor use_roll_17 = reshape(shape = var_1051, x = var_1050)[name = tensor("use_roll_17")]; - tensor var_1053 = sub(x = var_18, y = use_roll_17)[name = tensor("op_1053")]; - tensor var_1054 = mul(x = filled_window_17, y = var_1053)[name = tensor("op_1054")]; - tensor var_1055 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_1055")]; - tensor input_131 = add(x = var_1054, y = var_1055)[name = tensor("input_131")]; - tensor var_1057 = const()[name = tensor("op_1057"), val = tensor(0x1p+0)]; - tensor var_1058 = add(x = clip_15, y = var_1057)[name = tensor("op_1058")]; - tensor clip_17 = clip(alpha = var_22, beta = var_17, x = var_1058)[name = tensor("clip_17")]; - tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; - tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_131)[name = tensor("x_25")]; - tensor input_133_perm_0 = const()[name = tensor("input_133_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_55_pad_type_0 = const()[name = tensor("inputs_55_pad_type_0"), val = tensor("valid")]; - tensor inputs_55_strides_0 = const()[name = tensor("inputs_55_strides_0"), val = tensor([1])]; - tensor inputs_55_pad_0 = const()[name = tensor("inputs_55_pad_0"), val = tensor([0, 0])]; - tensor inputs_55_dilations_0 = const()[name = tensor("inputs_55_dilations_0"), val = tensor([1])]; - tensor inputs_55_groups_0 = const()[name = tensor("inputs_55_groups_0"), val = tensor(1)]; - tensor input_133 = transpose(perm = input_133_perm_0, x = x_25)[name = tensor("transpose_62")]; - tensor inputs_55 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_55_dilations_0, groups = inputs_55_groups_0, pad = inputs_55_pad_0, pad_type = inputs_55_pad_type_0, strides = inputs_55_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_133)[name = tensor("inputs_55")]; - tensor var_1079_split_sizes_0 = const()[name = tensor("op_1079_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1079_axis_0 = const()[name = tensor("op_1079_axis_0"), val = tensor(1)]; - tensor var_1079_0, tensor var_1079_1 = split(axis = var_1079_axis_0, split_sizes = var_1079_split_sizes_0, x = inputs_55)[name = tensor("op_1079")]; - tensor var_1081 = sigmoid(x = var_1079_1)[name = tensor("op_1081")]; - tensor inputs_57 = mul(x = var_1079_0, y = var_1081)[name = tensor("inputs_57")]; - tensor outputs_aug_17_pad_type_0 = const()[name = tensor("outputs_aug_17_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_17_pad_0 = const()[name = tensor("outputs_aug_17_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_17_groups_0 = const()[name = tensor("outputs_aug_17_groups_0"), val = tensor(256)]; - tensor outputs_aug_17_strides_0 = const()[name = tensor("outputs_aug_17_strides_0"), val = tensor([1])]; - tensor outputs_aug_17_dilations_0 = const()[name = tensor("outputs_aug_17_dilations_0"), val = tensor([1])]; - tensor outputs_aug_17 = conv(dilations = outputs_aug_17_dilations_0, groups = outputs_aug_17_groups_0, pad = outputs_aug_17_pad_0, pad_type = outputs_aug_17_pad_type_0, strides = outputs_aug_17_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_57)[name = tensor("outputs_aug_17")]; - tensor input_135_begin_0 = const()[name = tensor("input_135_begin_0"), val = tensor([0, 0, 0])]; - tensor input_135_end_0 = const()[name = tensor("input_135_end_0"), val = tensor([1, 256, 16])]; - tensor input_135_end_mask_0 = const()[name = tensor("input_135_end_mask_0"), val = tensor([true, true, false])]; - tensor input_135 = slice_by_index(begin = input_135_begin_0, end = input_135_end_0, end_mask = input_135_end_mask_0, x = outputs_aug_17)[name = tensor("input_135")]; - tensor inputs_59 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_135)[name = tensor("inputs_59")]; - tensor input_137 = silu(x = inputs_59)[name = tensor("input_137")]; - tensor input_139_pad_type_0 = const()[name = tensor("input_139_pad_type_0"), val = tensor("valid")]; - tensor input_139_strides_0 = const()[name = tensor("input_139_strides_0"), val = tensor([1])]; - tensor input_139_pad_0 = const()[name = tensor("input_139_pad_0"), val = tensor([0, 0])]; - tensor input_139_dilations_0 = const()[name = tensor("input_139_dilations_0"), val = tensor([1])]; - tensor input_139_groups_0 = const()[name = tensor("input_139_groups_0"), val = tensor(1)]; - tensor input_139 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_139_dilations_0, groups = input_139_groups_0, pad = input_139_pad_0, pad_type = input_139_pad_type_0, strides = input_139_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_137)[name = tensor("input_139")]; - tensor conv_out_17_perm_0 = const()[name = tensor("conv_out_17_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1112_begin_0 = const()[name = tensor("op_1112_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1112_end_0 = const()[name = tensor("op_1112_end_0"), val = tensor([1, 16, 256])]; - tensor var_1112_end_mask_0 = const()[name = tensor("op_1112_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_17 = transpose(perm = conv_out_17_perm_0, x = input_139)[name = tensor("transpose_61")]; - tensor var_1112 = slice_by_index(begin = var_1112_begin_0, end = var_1112_end_0, end_mask = var_1112_end_mask_0, x = conv_out_17)[name = tensor("op_1112")]; - tensor var_1115_begin_0 = const()[name = tensor("op_1115_begin_0"), val = tensor([0, 4, 0])]; - tensor var_1115_end_0 = const()[name = tensor("op_1115_end_0"), val = tensor([1, 1, 256])]; - tensor var_1115_end_mask_0 = const()[name = tensor("op_1115_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1115 = slice_by_index(begin = var_1115_begin_0, end = var_1115_end_0, end_mask = var_1115_end_mask_0, x = x_17)[name = tensor("op_1115")]; - tensor var_1118_begin_0 = const()[name = tensor("op_1118_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1118_end_0 = const()[name = tensor("op_1118_end_0"), val = tensor([1, 16, 256])]; - tensor var_1118_end_mask_0 = const()[name = tensor("op_1118_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1118 = slice_by_index(begin = var_1118_begin_0, end = var_1118_end_0, end_mask = var_1118_end_mask_0, x = input_131)[name = tensor("op_1118")]; + tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_654)[name = tensor("x_expanded_17")]; + tensor var_668 = sub(x = var_18, y = fill_mask_17)[name = tensor("op_668")]; + tensor var_669 = mul(x = window_19, y = var_668)[name = tensor("op_669")]; + tensor var_670 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_670")]; + tensor filled_window_17 = add(x = var_669, y = var_670)[name = tensor("filled_window_17")]; + tensor var_672 = greater_equal(x = clip_15, y = var_17)[name = tensor("op_672")]; + tensor var_673_dtype_0 = const()[name = tensor("op_673_dtype_0"), val = tensor("fp32")]; + tensor var_674 = const()[name = tensor("op_674"), val = tensor([1, 1, 1])]; + tensor var_673 = cast(dtype = var_673_dtype_0, x = var_672)[name = tensor("cast_174")]; + tensor use_roll_17 = reshape(shape = var_674, x = var_673)[name = tensor("use_roll_17")]; + tensor var_676 = sub(x = var_18, y = use_roll_17)[name = tensor("op_676")]; + tensor var_677 = mul(x = filled_window_17, y = var_676)[name = tensor("op_677")]; + tensor var_678 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_678")]; + tensor window_21 = add(x = var_677, y = var_678)[name = tensor("window_21")]; + tensor var_680 = const()[name = tensor("op_680"), val = tensor(0x1p+0)]; + tensor var_681 = add(x = clip_15, y = var_680)[name = tensor("op_681")]; + tensor clip_17 = clip(alpha = var_22, beta = var_17, x = var_681)[name = tensor("clip_17")]; + tensor var_684_begin_0 = const()[name = tensor("op_684_begin_0"), val = tensor([0, 4, 0])]; + tensor var_684_end_0 = const()[name = tensor("op_684_end_0"), val = tensor([1, 1, 256])]; + tensor var_684_end_mask_0 = const()[name = tensor("op_684_end_mask_0"), val = tensor([true, true, true])]; + tensor var_684 = slice_by_index(begin = var_684_begin_0, end = var_684_end_0, end_mask = var_684_end_mask_0, x = x_9)[name = tensor("op_684")]; + tensor var_687_begin_0 = const()[name = tensor("op_687_begin_0"), val = tensor([0, 1, 0])]; + tensor var_687_end_0 = const()[name = tensor("op_687_end_0"), val = tensor([1, 16, 256])]; + tensor var_687_end_mask_0 = const()[name = tensor("op_687_end_mask_0"), val = tensor([true, true, true])]; + tensor var_687 = slice_by_index(begin = var_687_begin_0, end = var_687_end_0, end_mask = var_687_end_mask_0, x = window_21)[name = tensor("op_687")]; tensor rolled_19_interleave_0 = const()[name = tensor("rolled_19_interleave_0"), val = tensor(false)]; - tensor rolled_19 = concat(axis = var_34, interleave = rolled_19_interleave_0, values = (var_1118, var_1115))[name = tensor("rolled_19")]; + tensor rolled_19 = concat(axis = var_34, interleave = rolled_19_interleave_0, values = (var_687, var_684))[name = tensor("rolled_19")]; tensor clip_18 = clip(alpha = var_22, beta = var_21, x = clip_17)[name = tensor("clip_18")]; - tensor var_1123 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_1123")]; - tensor var_1124_dtype_0 = const()[name = tensor("op_1124_dtype_0"), val = tensor("fp32")]; - tensor var_1125 = const()[name = tensor("op_1125"), val = tensor([1, 16, 1])]; - tensor var_1124 = cast(dtype = var_1124_dtype_0, x = var_1123)[name = tensor("cast_173")]; - tensor fill_mask_19 = reshape(shape = var_1125, x = var_1124)[name = tensor("fill_mask_19")]; + tensor var_692 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_692")]; + tensor var_693_dtype_0 = const()[name = tensor("op_693_dtype_0"), val = tensor("fp32")]; + tensor var_694 = const()[name = tensor("op_694"), val = tensor([1, 16, 1])]; + tensor var_693 = cast(dtype = var_693_dtype_0, x = var_692)[name = tensor("cast_173")]; + tensor fill_mask_19 = reshape(shape = var_694, x = var_693)[name = tensor("fill_mask_19")]; tensor x_expanded_19_reps_0 = const()[name = tensor("x_expanded_19_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_1115)[name = tensor("x_expanded_19")]; - tensor var_1129 = sub(x = var_18, y = fill_mask_19)[name = tensor("op_1129")]; - tensor var_1130 = mul(x = input_131, y = var_1129)[name = tensor("op_1130")]; - tensor var_1131 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_1131")]; - tensor filled_window_19 = add(x = var_1130, y = var_1131)[name = tensor("filled_window_19")]; - tensor var_1133 = greater_equal(x = clip_17, y = var_17)[name = tensor("op_1133")]; - tensor var_1134_dtype_0 = const()[name = tensor("op_1134_dtype_0"), val = tensor("fp32")]; - tensor var_1135 = const()[name = tensor("op_1135"), val = tensor([1, 1, 1])]; - tensor var_1134 = cast(dtype = var_1134_dtype_0, x = var_1133)[name = tensor("cast_172")]; - tensor use_roll_19 = reshape(shape = var_1135, x = var_1134)[name = tensor("use_roll_19")]; - tensor var_1137 = sub(x = var_18, y = use_roll_19)[name = tensor("op_1137")]; - tensor var_1138 = mul(x = filled_window_19, y = var_1137)[name = tensor("op_1138")]; - tensor var_1139 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_1139")]; - tensor input_141 = add(x = var_1138, y = var_1139)[name = tensor("input_141")]; - tensor var_1141 = const()[name = tensor("op_1141"), val = tensor(0x1p+0)]; - tensor var_1142 = add(x = clip_17, y = var_1141)[name = tensor("op_1142")]; - tensor clip_19 = clip(alpha = var_22, beta = var_17, x = var_1142)[name = tensor("clip_19")]; - tensor x_27_axes_0 = const()[name = tensor("x_27_axes_0"), val = tensor([-1])]; - tensor x_27 = layer_norm(axes = x_27_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_141)[name = tensor("x_27")]; - tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_61_pad_type_0 = const()[name = tensor("inputs_61_pad_type_0"), val = tensor("valid")]; - tensor inputs_61_strides_0 = const()[name = tensor("inputs_61_strides_0"), val = tensor([1])]; - tensor inputs_61_pad_0 = const()[name = tensor("inputs_61_pad_0"), val = tensor([0, 0])]; - tensor inputs_61_dilations_0 = const()[name = tensor("inputs_61_dilations_0"), val = tensor([1])]; - tensor inputs_61_groups_0 = const()[name = tensor("inputs_61_groups_0"), val = tensor(1)]; - tensor input_143 = transpose(perm = input_143_perm_0, x = x_27)[name = tensor("transpose_60")]; - tensor inputs_61 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_61_dilations_0, groups = inputs_61_groups_0, pad = inputs_61_pad_0, pad_type = inputs_61_pad_type_0, strides = inputs_61_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_61")]; - tensor var_1163_split_sizes_0 = const()[name = tensor("op_1163_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1163_axis_0 = const()[name = tensor("op_1163_axis_0"), val = tensor(1)]; - tensor var_1163_0, tensor var_1163_1 = split(axis = var_1163_axis_0, split_sizes = var_1163_split_sizes_0, x = inputs_61)[name = tensor("op_1163")]; - tensor var_1165 = sigmoid(x = var_1163_1)[name = tensor("op_1165")]; - tensor inputs_63 = mul(x = var_1163_0, y = var_1165)[name = tensor("inputs_63")]; - tensor outputs_aug_19_pad_type_0 = const()[name = tensor("outputs_aug_19_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_19_pad_0 = const()[name = tensor("outputs_aug_19_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_19_groups_0 = const()[name = tensor("outputs_aug_19_groups_0"), val = tensor(256)]; - tensor outputs_aug_19_strides_0 = const()[name = tensor("outputs_aug_19_strides_0"), val = tensor([1])]; - tensor outputs_aug_19_dilations_0 = const()[name = tensor("outputs_aug_19_dilations_0"), val = tensor([1])]; - tensor outputs_aug_19 = conv(dilations = outputs_aug_19_dilations_0, groups = outputs_aug_19_groups_0, pad = outputs_aug_19_pad_0, pad_type = outputs_aug_19_pad_type_0, strides = outputs_aug_19_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_63)[name = tensor("outputs_aug_19")]; - tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; - tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([1, 256, 16])]; - tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; - tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug_19)[name = tensor("input_145")]; - tensor inputs_65 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_145)[name = tensor("inputs_65")]; - tensor input_147 = silu(x = inputs_65)[name = tensor("input_147")]; - tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; - tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; - tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; - tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; - tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; - tensor input_149 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; - tensor conv_out_19_perm_0 = const()[name = tensor("conv_out_19_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1196_begin_0 = const()[name = tensor("op_1196_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1196_end_0 = const()[name = tensor("op_1196_end_0"), val = tensor([1, 16, 256])]; - tensor var_1196_end_mask_0 = const()[name = tensor("op_1196_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_19 = transpose(perm = conv_out_19_perm_0, x = input_149)[name = tensor("transpose_59")]; - tensor var_1196 = slice_by_index(begin = var_1196_begin_0, end = var_1196_end_0, end_mask = var_1196_end_mask_0, x = conv_out_19)[name = tensor("op_1196")]; - tensor var_1199_interleave_0 = const()[name = tensor("op_1199_interleave_0"), val = tensor(false)]; - tensor var_1199 = concat(axis = var_34, interleave = var_1199_interleave_0, values = (var_860, var_944, var_1028, var_1112, var_1196))[name = tensor("op_1199")]; - tensor input_151 = add(x = x_17, y = var_1199)[name = tensor("input_151")]; - tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; - tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; - tensor inputs_67 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_17")]; - tensor input_155 = silu(x = inputs_67)[name = tensor("input_155")]; - tensor input_159 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_18")]; - tensor var_1222 = const()[name = tensor("op_1222"), val = tensor(0x1p-1)]; - tensor var_1223 = mul(x = input_159, y = var_1222)[name = tensor("op_1223")]; - tensor input_161 = add(x = var_1223, y = input_151)[name = tensor("input_161")]; - tensor input_163_axes_0 = const()[name = tensor("input_163_axes_0"), val = tensor([-1])]; - tensor input_163 = layer_norm(axes = input_163_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_36, gamma = encoder_layer_norm_1_weight, x = input_161)[name = tensor("input_163")]; - tensor input_165_axes_0 = const()[name = tensor("input_165_axes_0"), val = tensor([-1])]; - tensor input_165 = layer_norm(axes = input_165_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_163)[name = tensor("input_165")]; - tensor inputs_69 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_165)[name = tensor("linear_19")]; - tensor input_167 = silu(x = inputs_69)[name = tensor("input_167")]; - tensor input_171 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_167)[name = tensor("linear_20")]; - tensor var_1252 = const()[name = tensor("op_1252"), val = tensor(0x1p-1)]; - tensor var_1253 = mul(x = input_171, y = var_1252)[name = tensor("op_1253")]; - tensor input_173 = add(x = var_1253, y = input_163)[name = tensor("input_173")]; - tensor x_29_axes_0 = const()[name = tensor("x_29_axes_0"), val = tensor([-1])]; - tensor x_29 = layer_norm(axes = x_29_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_36, gamma = encoder_ret_lns_2_weight, x = input_173)[name = tensor("x_29")]; + tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_684)[name = tensor("x_expanded_19")]; + tensor var_698 = sub(x = var_18, y = fill_mask_19)[name = tensor("op_698")]; + tensor var_699 = mul(x = window_21, y = var_698)[name = tensor("op_699")]; + tensor var_700 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_700")]; + tensor filled_window_19 = add(x = var_699, y = var_700)[name = tensor("filled_window_19")]; + tensor var_702 = greater_equal(x = clip_17, y = var_17)[name = tensor("op_702")]; + tensor var_703_dtype_0 = const()[name = tensor("op_703_dtype_0"), val = tensor("fp32")]; + tensor var_704 = const()[name = tensor("op_704"), val = tensor([1, 1, 1])]; + tensor var_703 = cast(dtype = var_703_dtype_0, x = var_702)[name = tensor("cast_172")]; + tensor use_roll_19 = reshape(shape = var_704, x = var_703)[name = tensor("use_roll_19")]; + tensor var_706 = sub(x = var_18, y = use_roll_19)[name = tensor("op_706")]; + tensor var_707 = mul(x = filled_window_19, y = var_706)[name = tensor("op_707")]; + tensor var_708 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_708")]; + tensor window_23 = add(x = var_707, y = var_708)[name = tensor("window_23")]; + tensor var_710 = const()[name = tensor("op_710"), val = tensor(0x1p+0)]; + tensor var_711 = add(x = clip_17, y = var_710)[name = tensor("op_711")]; + tensor clip_19 = clip(alpha = var_22, beta = var_17, x = var_711)[name = tensor("clip_19")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_31, interleave = input_61_interleave_0, values = (window_15, window_17, window_19, window_21, window_23))[name = tensor("input_61")]; + tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; + tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; + tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_13_pad_type_0 = const()[name = tensor("inputs_13_pad_type_0"), val = tensor("valid")]; + tensor inputs_13_strides_0 = const()[name = tensor("inputs_13_strides_0"), val = tensor([1])]; + tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; + tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; + tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; + tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; + tensor var_734_split_sizes_0 = const()[name = tensor("op_734_split_sizes_0"), val = tensor([256, 256])]; + tensor var_734_axis_0 = const()[name = tensor("op_734_axis_0"), val = tensor(1)]; + tensor var_734_0, tensor var_734_1 = split(axis = var_734_axis_0, split_sizes = var_734_split_sizes_0, x = inputs_13)[name = tensor("op_734")]; + tensor var_736 = sigmoid(x = var_734_1)[name = tensor("op_736")]; + tensor inputs_15 = mul(x = var_734_0, y = var_736)[name = tensor("inputs_15")]; + tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; + tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; + tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; + tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_15)[name = tensor("outputs_aug_3")]; + tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; + tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([5, 256, 16])]; + tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; + tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_3)[name = tensor("input_65")]; + tensor inputs_17 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_65)[name = tensor("inputs_17")]; + tensor input_67 = silu(x = inputs_17)[name = tensor("input_67")]; + tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; + tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; + tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; + tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; + tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; + tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; + tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; + tensor var_767_begin_0 = const()[name = tensor("op_767_begin_0"), val = tensor([0, -1, 0])]; + tensor var_767_end_0 = const()[name = tensor("op_767_end_0"), val = tensor([5, 16, 256])]; + tensor var_767_end_mask_0 = const()[name = tensor("op_767_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_767 = slice_by_index(begin = var_767_begin_0, end = var_767_end_0, end_mask = var_767_end_mask_0, x = conv_out_3)[name = tensor("op_767")]; + tensor var_769_perm_0 = const()[name = tensor("op_769_perm_0"), val = tensor([1, 0, 2])]; + tensor var_769 = transpose(perm = var_769_perm_0, x = var_767)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_769)[name = tensor("input_71")]; + tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; + tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; + tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; + tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; + tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; + tensor var_792 = const()[name = tensor("op_792"), val = tensor(0x1p-1)]; + tensor var_793 = mul(x = input_79, y = var_792)[name = tensor("op_793")]; + tensor input_81 = add(x = var_793, y = input_71)[name = tensor("input_81")]; + tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; + tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_36, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; + tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; + tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; + tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; + tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; + tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; + tensor var_822 = const()[name = tensor("op_822"), val = tensor(0x1p-1)]; + tensor var_823 = mul(x = input_91, y = var_822)[name = tensor("op_823")]; + tensor input_93 = add(x = var_823, y = input_83)[name = tensor("input_93")]; + tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; + tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_36, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; tensor prev_kv_5_end_0 = const()[name = tensor("prev_kv_5_end_0"), val = tensor([3, 1, 4, 64, 64])]; tensor prev_kv_5_end_mask_0 = const()[name = tensor("prev_kv_5_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1175,462 +868,308 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_1267 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_29)[name = tensor("linear_21")]; - tensor var_1268 = const()[name = tensor("op_1268"), val = tensor([1, 5, 4, 64])]; - tensor var_1269 = reshape(shape = var_1268, x = var_1267)[name = tensor("op_1269")]; + tensor var_837 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_838 = const()[name = tensor("op_838"), val = tensor([1, 5, 4, 64])]; + tensor var_839 = reshape(shape = var_838, x = var_837)[name = tensor("op_839")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1273 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_29)[name = tensor("linear_22")]; - tensor var_1274 = const()[name = tensor("op_1274"), val = tensor(0x1p-3)]; - tensor var_1275 = mul(x = var_1273, y = var_1274)[name = tensor("op_1275")]; - tensor var_1276 = const()[name = tensor("op_1276"), val = tensor([1, 5, 4, 64])]; - tensor var_1277 = reshape(shape = var_1276, x = var_1275)[name = tensor("op_1277")]; + tensor var_843 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_844 = const()[name = tensor("op_844"), val = tensor(0x1p-3)]; + tensor var_845 = mul(x = var_843, y = var_844)[name = tensor("op_845")]; + tensor var_846 = const()[name = tensor("op_846"), val = tensor([1, 5, 4, 64])]; + tensor var_847 = reshape(shape = var_846, x = var_845)[name = tensor("op_847")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1281 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_29)[name = tensor("linear_23")]; - tensor var_1282 = const()[name = tensor("op_1282"), val = tensor([1, 5, 4, 64])]; - tensor var_1283 = reshape(shape = var_1282, x = var_1281)[name = tensor("op_1283")]; + tensor var_851 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_852 = const()[name = tensor("op_852"), val = tensor([1, 5, 4, 64])]; + tensor var_853 = reshape(shape = var_852, x = var_851)[name = tensor("op_853")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_177 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_29)[name = tensor("linear_24")]; + tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; tensor s_t_5 = add(x = prev_scale_5, y = encoder__t_index)[name = tensor("s_t_5")]; tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_1277)[name = tensor("transpose_57")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_1269)[name = tensor("transpose_58")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_847)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_839)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_1293 = const()[name = tensor("op_1293"), val = tensor([5, 1])]; - tensor var_1294 = reshape(shape = var_1293, x = sqrt_s_t_5)[name = tensor("op_1294")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_1294)[name = tensor("M_5")]; - tensor var_1296 = mul(x = qk_5, y = M_5)[name = tensor("op_1296")]; + tensor var_863 = const()[name = tensor("op_863"), val = tensor([5, 1])]; + tensor var_864 = reshape(shape = var_863, x = sqrt_s_t_5)[name = tensor("op_864")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_864)[name = tensor("M_5")]; + tensor var_866 = mul(x = qk_5, y = M_5)[name = tensor("op_866")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_1283)[name = tensor("transpose_56")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_1296, y = v_5)[name = tensor("inner_5")]; - tensor var_1298_transpose_x_0 = const()[name = tensor("op_1298_transpose_x_0"), val = tensor(false)]; - tensor var_1298_transpose_y_0 = const()[name = tensor("op_1298_transpose_y_0"), val = tensor(false)]; - tensor var_1298 = matmul(transpose_x = var_1298_transpose_x_0, transpose_y = var_1298_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_1298")]; - tensor var_1299 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_1299")]; - tensor var_1300 = const()[name = tensor("op_1300"), val = tensor([1, 1, 5, 1])]; - tensor var_1301 = reshape(shape = var_1300, x = var_1299)[name = tensor("op_1301")]; - tensor cross_5 = mul(x = var_1298, y = var_1301)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_853)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_866, y = v_5)[name = tensor("inner_5")]; + tensor var_868_transpose_x_0 = const()[name = tensor("op_868_transpose_x_0"), val = tensor(false)]; + tensor var_868_transpose_y_0 = const()[name = tensor("op_868_transpose_y_0"), val = tensor(false)]; + tensor var_868 = matmul(transpose_x = var_868_transpose_x_0, transpose_y = var_868_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_868")]; + tensor var_869 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_869")]; + tensor var_870 = const()[name = tensor("op_870"), val = tensor([1, 1, 5, 1])]; + tensor var_871 = reshape(shape = var_870, x = var_869)[name = tensor("op_871")]; + tensor cross_5 = mul(x = var_868, y = var_871)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_1304 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_1304")]; - tensor var_1306_transpose_x_1 = const()[name = tensor("op_1306_transpose_x_1"), val = tensor(true)]; - tensor var_1306_transpose_y_1 = const()[name = tensor("op_1306_transpose_y_1"), val = tensor(false)]; - tensor var_1306 = matmul(transpose_x = var_1306_transpose_x_1, transpose_y = var_1306_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_1306")]; - tensor new_kv_unnorm_5 = add(x = var_1304, y = var_1306)[name = tensor("new_kv_unnorm_5")]; - tensor var_1308 = const()[name = tensor("op_1308"), val = tensor(0x1.4p+2)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_1308)[name = tensor("new_scale_5")]; - tensor var_1310 = sqrt(x = new_scale_5)[name = tensor("op_1310")]; - tensor var_1311 = real_div(x = new_kv_unnorm_5, y = var_1310)[name = tensor("op_1311")]; - tensor var_1312_perm_0 = const()[name = tensor("op_1312_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_874 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_874")]; + tensor var_876_transpose_x_1 = const()[name = tensor("op_876_transpose_x_1"), val = tensor(true)]; + tensor var_876_transpose_y_1 = const()[name = tensor("op_876_transpose_y_1"), val = tensor(false)]; + tensor var_876 = matmul(transpose_x = var_876_transpose_x_1, transpose_y = var_876_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_876")]; + tensor new_kv_unnorm_5 = add(x = var_874, y = var_876)[name = tensor("new_kv_unnorm_5")]; + tensor var_878 = const()[name = tensor("op_878"), val = tensor(0x1.4p+2)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_878)[name = tensor("new_scale_5")]; + tensor var_880 = sqrt(x = new_scale_5)[name = tensor("op_880")]; + tensor var_881 = real_div(x = new_kv_unnorm_5, y = var_880)[name = tensor("op_881")]; + tensor var_882_perm_0 = const()[name = tensor("op_882_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_1312 = transpose(perm = var_1312_perm_0, x = out_13)[name = tensor("transpose_55")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_25, x = var_1312)[name = tensor("out_15")]; - tensor var_1316 = const()[name = tensor("op_1316"), val = tensor([1, 5, 256])]; - tensor out_17 = reshape(shape = var_1316, x = out_15)[name = tensor("out_17")]; - tensor var_1318 = silu(x = input_177)[name = tensor("op_1318")]; - tensor input_179 = mul(x = var_1318, y = out_17)[name = tensor("input_179")]; - tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_179)[name = tensor("linear_25")]; - tensor x_31 = add(x = input_173, y = ret_out_5)[name = tensor("x_31")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor var_882 = transpose(perm = var_882_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_25, x = var_882)[name = tensor("out_15")]; + tensor var_886 = const()[name = tensor("op_886"), val = tensor([1, 5, 256])]; + tensor out_17 = reshape(shape = var_886, x = out_15)[name = tensor("out_17")]; + tensor var_888 = silu(x = input_97)[name = tensor("op_888")]; + tensor input_99 = mul(x = var_888, y = out_17)[name = tensor("input_99")]; + tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; + tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; + tensor window_25_begin_0 = const()[name = tensor("window_25_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_25_end_0 = const()[name = tensor("window_25_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_25_end_mask_0 = const()[name = tensor("window_25_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_25_squeeze_mask_0 = const()[name = tensor("window_25_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_25 = slice_by_index(begin = window_25_begin_0, end = window_25_end_0, end_mask = window_25_end_mask_0, squeeze_mask = window_25_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_25")]; tensor filled_21_begin_0 = const()[name = tensor("filled_21_begin_0"), val = tensor([2, 0])]; tensor filled_21_end_0 = const()[name = tensor("filled_21_end_0"), val = tensor([3, 1])]; tensor filled_21_end_mask_0 = const()[name = tensor("filled_21_end_mask_0"), val = tensor([false, true])]; tensor filled_21_squeeze_mask_0 = const()[name = tensor("filled_21_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_21 = slice_by_index(begin = filled_21_begin_0, end = filled_21_end_0, end_mask = filled_21_end_mask_0, squeeze_mask = filled_21_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_21")]; - tensor var_1327_begin_0 = const()[name = tensor("op_1327_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1327_end_0 = const()[name = tensor("op_1327_end_0"), val = tensor([1, 1, 256])]; - tensor var_1327_end_mask_0 = const()[name = tensor("op_1327_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1327 = slice_by_index(begin = var_1327_begin_0, end = var_1327_end_0, end_mask = var_1327_end_mask_0, x = x_31)[name = tensor("op_1327")]; - tensor var_1330_begin_0 = const()[name = tensor("op_1330_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1330_end_0 = const()[name = tensor("op_1330_end_0"), val = tensor([1, 16, 256])]; - tensor var_1330_end_mask_0 = const()[name = tensor("op_1330_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1330 = slice_by_index(begin = var_1330_begin_0, end = var_1330_end_0, end_mask = var_1330_end_mask_0, x = window_5)[name = tensor("op_1330")]; + tensor var_897_begin_0 = const()[name = tensor("op_897_begin_0"), val = tensor([0, 0, 0])]; + tensor var_897_end_0 = const()[name = tensor("op_897_end_0"), val = tensor([1, 1, 256])]; + tensor var_897_end_mask_0 = const()[name = tensor("op_897_end_mask_0"), val = tensor([true, false, true])]; + tensor var_897 = slice_by_index(begin = var_897_begin_0, end = var_897_end_0, end_mask = var_897_end_mask_0, x = x_15)[name = tensor("op_897")]; + tensor var_900_begin_0 = const()[name = tensor("op_900_begin_0"), val = tensor([0, 1, 0])]; + tensor var_900_end_0 = const()[name = tensor("op_900_end_0"), val = tensor([1, 16, 256])]; + tensor var_900_end_mask_0 = const()[name = tensor("op_900_end_mask_0"), val = tensor([true, true, true])]; + tensor var_900 = slice_by_index(begin = var_900_begin_0, end = var_900_end_0, end_mask = var_900_end_mask_0, x = window_25)[name = tensor("op_900")]; tensor rolled_21_interleave_0 = const()[name = tensor("rolled_21_interleave_0"), val = tensor(false)]; - tensor rolled_21 = concat(axis = var_34, interleave = rolled_21_interleave_0, values = (var_1330, var_1327))[name = tensor("rolled_21")]; + tensor rolled_21 = concat(axis = var_34, interleave = rolled_21_interleave_0, values = (var_900, var_897))[name = tensor("rolled_21")]; tensor clip_20 = clip(alpha = var_22, beta = var_21, x = filled_21)[name = tensor("clip_20")]; - tensor var_1335 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_1335")]; - tensor var_1336_dtype_0 = const()[name = tensor("op_1336_dtype_0"), val = tensor("fp32")]; - tensor var_1337 = const()[name = tensor("op_1337"), val = tensor([1, 16, 1])]; - tensor var_1336 = cast(dtype = var_1336_dtype_0, x = var_1335)[name = tensor("cast_171")]; - tensor fill_mask_21 = reshape(shape = var_1337, x = var_1336)[name = tensor("fill_mask_21")]; + tensor var_905 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_905")]; + tensor var_906_dtype_0 = const()[name = tensor("op_906_dtype_0"), val = tensor("fp32")]; + tensor var_907 = const()[name = tensor("op_907"), val = tensor([1, 16, 1])]; + tensor var_906 = cast(dtype = var_906_dtype_0, x = var_905)[name = tensor("cast_171")]; + tensor fill_mask_21 = reshape(shape = var_907, x = var_906)[name = tensor("fill_mask_21")]; tensor x_expanded_21_reps_0 = const()[name = tensor("x_expanded_21_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_1327)[name = tensor("x_expanded_21")]; - tensor var_1341 = sub(x = var_18, y = fill_mask_21)[name = tensor("op_1341")]; - tensor var_1342 = mul(x = window_5, y = var_1341)[name = tensor("op_1342")]; - tensor var_1343 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_1343")]; - tensor filled_window_21 = add(x = var_1342, y = var_1343)[name = tensor("filled_window_21")]; - tensor var_1345 = greater_equal(x = filled_21, y = var_17)[name = tensor("op_1345")]; - tensor var_1346_dtype_0 = const()[name = tensor("op_1346_dtype_0"), val = tensor("fp32")]; - tensor var_1347 = const()[name = tensor("op_1347"), val = tensor([1, 1, 1])]; - tensor var_1346 = cast(dtype = var_1346_dtype_0, x = var_1345)[name = tensor("cast_170")]; - tensor use_roll_21 = reshape(shape = var_1347, x = var_1346)[name = tensor("use_roll_21")]; - tensor var_1349 = sub(x = var_18, y = use_roll_21)[name = tensor("op_1349")]; - tensor var_1350 = mul(x = filled_window_21, y = var_1349)[name = tensor("op_1350")]; - tensor var_1351 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_1351")]; - tensor input_181 = add(x = var_1350, y = var_1351)[name = tensor("input_181")]; - tensor var_1353 = const()[name = tensor("op_1353"), val = tensor(0x1p+0)]; - tensor var_1354 = add(x = filled_21, y = var_1353)[name = tensor("op_1354")]; - tensor clip_21 = clip(alpha = var_22, beta = var_17, x = var_1354)[name = tensor("clip_21")]; - tensor x_33_axes_0 = const()[name = tensor("x_33_axes_0"), val = tensor([-1])]; - tensor x_33 = layer_norm(axes = x_33_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_181)[name = tensor("x_33")]; - tensor input_183_perm_0 = const()[name = tensor("input_183_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_71_pad_type_0 = const()[name = tensor("inputs_71_pad_type_0"), val = tensor("valid")]; - tensor inputs_71_strides_0 = const()[name = tensor("inputs_71_strides_0"), val = tensor([1])]; - tensor inputs_71_pad_0 = const()[name = tensor("inputs_71_pad_0"), val = tensor([0, 0])]; - tensor inputs_71_dilations_0 = const()[name = tensor("inputs_71_dilations_0"), val = tensor([1])]; - tensor inputs_71_groups_0 = const()[name = tensor("inputs_71_groups_0"), val = tensor(1)]; - tensor input_183 = transpose(perm = input_183_perm_0, x = x_33)[name = tensor("transpose_54")]; - tensor inputs_71 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_71_dilations_0, groups = inputs_71_groups_0, pad = inputs_71_pad_0, pad_type = inputs_71_pad_type_0, strides = inputs_71_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_183)[name = tensor("inputs_71")]; - tensor var_1375_split_sizes_0 = const()[name = tensor("op_1375_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1375_axis_0 = const()[name = tensor("op_1375_axis_0"), val = tensor(1)]; - tensor var_1375_0, tensor var_1375_1 = split(axis = var_1375_axis_0, split_sizes = var_1375_split_sizes_0, x = inputs_71)[name = tensor("op_1375")]; - tensor var_1377 = sigmoid(x = var_1375_1)[name = tensor("op_1377")]; - tensor inputs_73 = mul(x = var_1375_0, y = var_1377)[name = tensor("inputs_73")]; - tensor outputs_aug_21_pad_type_0 = const()[name = tensor("outputs_aug_21_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_21_pad_0 = const()[name = tensor("outputs_aug_21_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_21_groups_0 = const()[name = tensor("outputs_aug_21_groups_0"), val = tensor(256)]; - tensor outputs_aug_21_strides_0 = const()[name = tensor("outputs_aug_21_strides_0"), val = tensor([1])]; - tensor outputs_aug_21_dilations_0 = const()[name = tensor("outputs_aug_21_dilations_0"), val = tensor([1])]; - tensor outputs_aug_21 = conv(dilations = outputs_aug_21_dilations_0, groups = outputs_aug_21_groups_0, pad = outputs_aug_21_pad_0, pad_type = outputs_aug_21_pad_type_0, strides = outputs_aug_21_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_73)[name = tensor("outputs_aug_21")]; - tensor input_185_begin_0 = const()[name = tensor("input_185_begin_0"), val = tensor([0, 0, 0])]; - tensor input_185_end_0 = const()[name = tensor("input_185_end_0"), val = tensor([1, 256, 16])]; - tensor input_185_end_mask_0 = const()[name = tensor("input_185_end_mask_0"), val = tensor([true, true, false])]; - tensor input_185 = slice_by_index(begin = input_185_begin_0, end = input_185_end_0, end_mask = input_185_end_mask_0, x = outputs_aug_21)[name = tensor("input_185")]; - tensor inputs_75 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_185)[name = tensor("inputs_75")]; - tensor input_187 = silu(x = inputs_75)[name = tensor("input_187")]; - tensor input_189_pad_type_0 = const()[name = tensor("input_189_pad_type_0"), val = tensor("valid")]; - tensor input_189_strides_0 = const()[name = tensor("input_189_strides_0"), val = tensor([1])]; - tensor input_189_pad_0 = const()[name = tensor("input_189_pad_0"), val = tensor([0, 0])]; - tensor input_189_dilations_0 = const()[name = tensor("input_189_dilations_0"), val = tensor([1])]; - tensor input_189_groups_0 = const()[name = tensor("input_189_groups_0"), val = tensor(1)]; - tensor input_189 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_189_dilations_0, groups = input_189_groups_0, pad = input_189_pad_0, pad_type = input_189_pad_type_0, strides = input_189_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_187)[name = tensor("input_189")]; - tensor conv_out_21_perm_0 = const()[name = tensor("conv_out_21_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1408_begin_0 = const()[name = tensor("op_1408_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1408_end_0 = const()[name = tensor("op_1408_end_0"), val = tensor([1, 16, 256])]; - tensor var_1408_end_mask_0 = const()[name = tensor("op_1408_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_21 = transpose(perm = conv_out_21_perm_0, x = input_189)[name = tensor("transpose_53")]; - tensor var_1408 = slice_by_index(begin = var_1408_begin_0, end = var_1408_end_0, end_mask = var_1408_end_mask_0, x = conv_out_21)[name = tensor("op_1408")]; - tensor var_1411_begin_0 = const()[name = tensor("op_1411_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1411_end_0 = const()[name = tensor("op_1411_end_0"), val = tensor([1, 2, 256])]; - tensor var_1411_end_mask_0 = const()[name = tensor("op_1411_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1411 = slice_by_index(begin = var_1411_begin_0, end = var_1411_end_0, end_mask = var_1411_end_mask_0, x = x_31)[name = tensor("op_1411")]; - tensor var_1414_begin_0 = const()[name = tensor("op_1414_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1414_end_0 = const()[name = tensor("op_1414_end_0"), val = tensor([1, 16, 256])]; - tensor var_1414_end_mask_0 = const()[name = tensor("op_1414_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1414 = slice_by_index(begin = var_1414_begin_0, end = var_1414_end_0, end_mask = var_1414_end_mask_0, x = input_181)[name = tensor("op_1414")]; + tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_897)[name = tensor("x_expanded_21")]; + tensor var_911 = sub(x = var_18, y = fill_mask_21)[name = tensor("op_911")]; + tensor var_912 = mul(x = window_25, y = var_911)[name = tensor("op_912")]; + tensor var_913 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_913")]; + tensor filled_window_21 = add(x = var_912, y = var_913)[name = tensor("filled_window_21")]; + tensor var_915 = greater_equal(x = filled_21, y = var_17)[name = tensor("op_915")]; + tensor var_916_dtype_0 = const()[name = tensor("op_916_dtype_0"), val = tensor("fp32")]; + tensor var_917 = const()[name = tensor("op_917"), val = tensor([1, 1, 1])]; + tensor var_916 = cast(dtype = var_916_dtype_0, x = var_915)[name = tensor("cast_170")]; + tensor use_roll_21 = reshape(shape = var_917, x = var_916)[name = tensor("use_roll_21")]; + tensor var_919 = sub(x = var_18, y = use_roll_21)[name = tensor("op_919")]; + tensor var_920 = mul(x = filled_window_21, y = var_919)[name = tensor("op_920")]; + tensor var_921 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_921")]; + tensor window_27 = add(x = var_920, y = var_921)[name = tensor("window_27")]; + tensor var_923 = const()[name = tensor("op_923"), val = tensor(0x1p+0)]; + tensor var_924 = add(x = filled_21, y = var_923)[name = tensor("op_924")]; + tensor clip_21 = clip(alpha = var_22, beta = var_17, x = var_924)[name = tensor("clip_21")]; + tensor var_927_begin_0 = const()[name = tensor("op_927_begin_0"), val = tensor([0, 1, 0])]; + tensor var_927_end_0 = const()[name = tensor("op_927_end_0"), val = tensor([1, 2, 256])]; + tensor var_927_end_mask_0 = const()[name = tensor("op_927_end_mask_0"), val = tensor([true, false, true])]; + tensor var_927 = slice_by_index(begin = var_927_begin_0, end = var_927_end_0, end_mask = var_927_end_mask_0, x = x_15)[name = tensor("op_927")]; + tensor var_930_begin_0 = const()[name = tensor("op_930_begin_0"), val = tensor([0, 1, 0])]; + tensor var_930_end_0 = const()[name = tensor("op_930_end_0"), val = tensor([1, 16, 256])]; + tensor var_930_end_mask_0 = const()[name = tensor("op_930_end_mask_0"), val = tensor([true, true, true])]; + tensor var_930 = slice_by_index(begin = var_930_begin_0, end = var_930_end_0, end_mask = var_930_end_mask_0, x = window_27)[name = tensor("op_930")]; tensor rolled_23_interleave_0 = const()[name = tensor("rolled_23_interleave_0"), val = tensor(false)]; - tensor rolled_23 = concat(axis = var_34, interleave = rolled_23_interleave_0, values = (var_1414, var_1411))[name = tensor("rolled_23")]; + tensor rolled_23 = concat(axis = var_34, interleave = rolled_23_interleave_0, values = (var_930, var_927))[name = tensor("rolled_23")]; tensor clip_22 = clip(alpha = var_22, beta = var_21, x = clip_21)[name = tensor("clip_22")]; - tensor var_1419 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_1419")]; - tensor var_1420_dtype_0 = const()[name = tensor("op_1420_dtype_0"), val = tensor("fp32")]; - tensor var_1421 = const()[name = tensor("op_1421"), val = tensor([1, 16, 1])]; - tensor var_1420 = cast(dtype = var_1420_dtype_0, x = var_1419)[name = tensor("cast_169")]; - tensor fill_mask_23 = reshape(shape = var_1421, x = var_1420)[name = tensor("fill_mask_23")]; + tensor var_935 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_935")]; + tensor var_936_dtype_0 = const()[name = tensor("op_936_dtype_0"), val = tensor("fp32")]; + tensor var_937 = const()[name = tensor("op_937"), val = tensor([1, 16, 1])]; + tensor var_936 = cast(dtype = var_936_dtype_0, x = var_935)[name = tensor("cast_169")]; + tensor fill_mask_23 = reshape(shape = var_937, x = var_936)[name = tensor("fill_mask_23")]; tensor x_expanded_23_reps_0 = const()[name = tensor("x_expanded_23_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_23 = tile(reps = x_expanded_23_reps_0, x = var_1411)[name = tensor("x_expanded_23")]; - tensor var_1425 = sub(x = var_18, y = fill_mask_23)[name = tensor("op_1425")]; - tensor var_1426 = mul(x = input_181, y = var_1425)[name = tensor("op_1426")]; - tensor var_1427 = mul(x = x_expanded_23, y = fill_mask_23)[name = tensor("op_1427")]; - tensor filled_window_23 = add(x = var_1426, y = var_1427)[name = tensor("filled_window_23")]; - tensor var_1429 = greater_equal(x = clip_21, y = var_17)[name = tensor("op_1429")]; - tensor var_1430_dtype_0 = const()[name = tensor("op_1430_dtype_0"), val = tensor("fp32")]; - tensor var_1431 = const()[name = tensor("op_1431"), val = tensor([1, 1, 1])]; - tensor var_1430 = cast(dtype = var_1430_dtype_0, x = var_1429)[name = tensor("cast_168")]; - tensor use_roll_23 = reshape(shape = var_1431, x = var_1430)[name = tensor("use_roll_23")]; - tensor var_1433 = sub(x = var_18, y = use_roll_23)[name = tensor("op_1433")]; - tensor var_1434 = mul(x = filled_window_23, y = var_1433)[name = tensor("op_1434")]; - tensor var_1435 = mul(x = rolled_23, y = use_roll_23)[name = tensor("op_1435")]; - tensor input_191 = add(x = var_1434, y = var_1435)[name = tensor("input_191")]; - tensor var_1437 = const()[name = tensor("op_1437"), val = tensor(0x1p+0)]; - tensor var_1438 = add(x = clip_21, y = var_1437)[name = tensor("op_1438")]; - tensor clip_23 = clip(alpha = var_22, beta = var_17, x = var_1438)[name = tensor("clip_23")]; - tensor x_35_axes_0 = const()[name = tensor("x_35_axes_0"), val = tensor([-1])]; - tensor x_35 = layer_norm(axes = x_35_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_191)[name = tensor("x_35")]; - tensor input_193_perm_0 = const()[name = tensor("input_193_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_77_pad_type_0 = const()[name = tensor("inputs_77_pad_type_0"), val = tensor("valid")]; - tensor inputs_77_strides_0 = const()[name = tensor("inputs_77_strides_0"), val = tensor([1])]; - tensor inputs_77_pad_0 = const()[name = tensor("inputs_77_pad_0"), val = tensor([0, 0])]; - tensor inputs_77_dilations_0 = const()[name = tensor("inputs_77_dilations_0"), val = tensor([1])]; - tensor inputs_77_groups_0 = const()[name = tensor("inputs_77_groups_0"), val = tensor(1)]; - tensor input_193 = transpose(perm = input_193_perm_0, x = x_35)[name = tensor("transpose_52")]; - tensor inputs_77 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_77_dilations_0, groups = inputs_77_groups_0, pad = inputs_77_pad_0, pad_type = inputs_77_pad_type_0, strides = inputs_77_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_193)[name = tensor("inputs_77")]; - tensor var_1459_split_sizes_0 = const()[name = tensor("op_1459_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1459_axis_0 = const()[name = tensor("op_1459_axis_0"), val = tensor(1)]; - tensor var_1459_0, tensor var_1459_1 = split(axis = var_1459_axis_0, split_sizes = var_1459_split_sizes_0, x = inputs_77)[name = tensor("op_1459")]; - tensor var_1461 = sigmoid(x = var_1459_1)[name = tensor("op_1461")]; - tensor inputs_79 = mul(x = var_1459_0, y = var_1461)[name = tensor("inputs_79")]; - tensor outputs_aug_23_pad_type_0 = const()[name = tensor("outputs_aug_23_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_23_pad_0 = const()[name = tensor("outputs_aug_23_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_23_groups_0 = const()[name = tensor("outputs_aug_23_groups_0"), val = tensor(256)]; - tensor outputs_aug_23_strides_0 = const()[name = tensor("outputs_aug_23_strides_0"), val = tensor([1])]; - tensor outputs_aug_23_dilations_0 = const()[name = tensor("outputs_aug_23_dilations_0"), val = tensor([1])]; - tensor outputs_aug_23 = conv(dilations = outputs_aug_23_dilations_0, groups = outputs_aug_23_groups_0, pad = outputs_aug_23_pad_0, pad_type = outputs_aug_23_pad_type_0, strides = outputs_aug_23_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_79)[name = tensor("outputs_aug_23")]; - tensor input_195_begin_0 = const()[name = tensor("input_195_begin_0"), val = tensor([0, 0, 0])]; - tensor input_195_end_0 = const()[name = tensor("input_195_end_0"), val = tensor([1, 256, 16])]; - tensor input_195_end_mask_0 = const()[name = tensor("input_195_end_mask_0"), val = tensor([true, true, false])]; - tensor input_195 = slice_by_index(begin = input_195_begin_0, end = input_195_end_0, end_mask = input_195_end_mask_0, x = outputs_aug_23)[name = tensor("input_195")]; - tensor inputs_81 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_195)[name = tensor("inputs_81")]; - tensor input_197 = silu(x = inputs_81)[name = tensor("input_197")]; - tensor input_199_pad_type_0 = const()[name = tensor("input_199_pad_type_0"), val = tensor("valid")]; - tensor input_199_strides_0 = const()[name = tensor("input_199_strides_0"), val = tensor([1])]; - tensor input_199_pad_0 = const()[name = tensor("input_199_pad_0"), val = tensor([0, 0])]; - tensor input_199_dilations_0 = const()[name = tensor("input_199_dilations_0"), val = tensor([1])]; - tensor input_199_groups_0 = const()[name = tensor("input_199_groups_0"), val = tensor(1)]; - tensor input_199 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_199_dilations_0, groups = input_199_groups_0, pad = input_199_pad_0, pad_type = input_199_pad_type_0, strides = input_199_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_197)[name = tensor("input_199")]; - tensor conv_out_23_perm_0 = const()[name = tensor("conv_out_23_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1492_begin_0 = const()[name = tensor("op_1492_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1492_end_0 = const()[name = tensor("op_1492_end_0"), val = tensor([1, 16, 256])]; - tensor var_1492_end_mask_0 = const()[name = tensor("op_1492_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_23 = transpose(perm = conv_out_23_perm_0, x = input_199)[name = tensor("transpose_51")]; - tensor var_1492 = slice_by_index(begin = var_1492_begin_0, end = var_1492_end_0, end_mask = var_1492_end_mask_0, x = conv_out_23)[name = tensor("op_1492")]; - tensor var_1495_begin_0 = const()[name = tensor("op_1495_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1495_end_0 = const()[name = tensor("op_1495_end_0"), val = tensor([1, 3, 256])]; - tensor var_1495_end_mask_0 = const()[name = tensor("op_1495_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1495 = slice_by_index(begin = var_1495_begin_0, end = var_1495_end_0, end_mask = var_1495_end_mask_0, x = x_31)[name = tensor("op_1495")]; - tensor var_1498_begin_0 = const()[name = tensor("op_1498_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1498_end_0 = const()[name = tensor("op_1498_end_0"), val = tensor([1, 16, 256])]; - tensor var_1498_end_mask_0 = const()[name = tensor("op_1498_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1498 = slice_by_index(begin = var_1498_begin_0, end = var_1498_end_0, end_mask = var_1498_end_mask_0, x = input_191)[name = tensor("op_1498")]; + tensor x_expanded_23 = tile(reps = x_expanded_23_reps_0, x = var_927)[name = tensor("x_expanded_23")]; + tensor var_941 = sub(x = var_18, y = fill_mask_23)[name = tensor("op_941")]; + tensor var_942 = mul(x = window_27, y = var_941)[name = tensor("op_942")]; + tensor var_943 = mul(x = x_expanded_23, y = fill_mask_23)[name = tensor("op_943")]; + tensor filled_window_23 = add(x = var_942, y = var_943)[name = tensor("filled_window_23")]; + tensor var_945 = greater_equal(x = clip_21, y = var_17)[name = tensor("op_945")]; + tensor var_946_dtype_0 = const()[name = tensor("op_946_dtype_0"), val = tensor("fp32")]; + tensor var_947 = const()[name = tensor("op_947"), val = tensor([1, 1, 1])]; + tensor var_946 = cast(dtype = var_946_dtype_0, x = var_945)[name = tensor("cast_168")]; + tensor use_roll_23 = reshape(shape = var_947, x = var_946)[name = tensor("use_roll_23")]; + tensor var_949 = sub(x = var_18, y = use_roll_23)[name = tensor("op_949")]; + tensor var_950 = mul(x = filled_window_23, y = var_949)[name = tensor("op_950")]; + tensor var_951 = mul(x = rolled_23, y = use_roll_23)[name = tensor("op_951")]; + tensor window_29 = add(x = var_950, y = var_951)[name = tensor("window_29")]; + tensor var_953 = const()[name = tensor("op_953"), val = tensor(0x1p+0)]; + tensor var_954 = add(x = clip_21, y = var_953)[name = tensor("op_954")]; + tensor clip_23 = clip(alpha = var_22, beta = var_17, x = var_954)[name = tensor("clip_23")]; + tensor var_957_begin_0 = const()[name = tensor("op_957_begin_0"), val = tensor([0, 2, 0])]; + tensor var_957_end_0 = const()[name = tensor("op_957_end_0"), val = tensor([1, 3, 256])]; + tensor var_957_end_mask_0 = const()[name = tensor("op_957_end_mask_0"), val = tensor([true, false, true])]; + tensor var_957 = slice_by_index(begin = var_957_begin_0, end = var_957_end_0, end_mask = var_957_end_mask_0, x = x_15)[name = tensor("op_957")]; + tensor var_960_begin_0 = const()[name = tensor("op_960_begin_0"), val = tensor([0, 1, 0])]; + tensor var_960_end_0 = const()[name = tensor("op_960_end_0"), val = tensor([1, 16, 256])]; + tensor var_960_end_mask_0 = const()[name = tensor("op_960_end_mask_0"), val = tensor([true, true, true])]; + tensor var_960 = slice_by_index(begin = var_960_begin_0, end = var_960_end_0, end_mask = var_960_end_mask_0, x = window_29)[name = tensor("op_960")]; tensor rolled_25_interleave_0 = const()[name = tensor("rolled_25_interleave_0"), val = tensor(false)]; - tensor rolled_25 = concat(axis = var_34, interleave = rolled_25_interleave_0, values = (var_1498, var_1495))[name = tensor("rolled_25")]; + tensor rolled_25 = concat(axis = var_34, interleave = rolled_25_interleave_0, values = (var_960, var_957))[name = tensor("rolled_25")]; tensor clip_24 = clip(alpha = var_22, beta = var_21, x = clip_23)[name = tensor("clip_24")]; - tensor var_1503 = equal(x = encoder__conv_positions, y = clip_24)[name = tensor("op_1503")]; - tensor var_1504_dtype_0 = const()[name = tensor("op_1504_dtype_0"), val = tensor("fp32")]; - tensor var_1505 = const()[name = tensor("op_1505"), val = tensor([1, 16, 1])]; - tensor var_1504 = cast(dtype = var_1504_dtype_0, x = var_1503)[name = tensor("cast_167")]; - tensor fill_mask_25 = reshape(shape = var_1505, x = var_1504)[name = tensor("fill_mask_25")]; + tensor var_965 = equal(x = encoder__conv_positions, y = clip_24)[name = tensor("op_965")]; + tensor var_966_dtype_0 = const()[name = tensor("op_966_dtype_0"), val = tensor("fp32")]; + tensor var_967 = const()[name = tensor("op_967"), val = tensor([1, 16, 1])]; + tensor var_966 = cast(dtype = var_966_dtype_0, x = var_965)[name = tensor("cast_167")]; + tensor fill_mask_25 = reshape(shape = var_967, x = var_966)[name = tensor("fill_mask_25")]; tensor x_expanded_25_reps_0 = const()[name = tensor("x_expanded_25_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_25 = tile(reps = x_expanded_25_reps_0, x = var_1495)[name = tensor("x_expanded_25")]; - tensor var_1509 = sub(x = var_18, y = fill_mask_25)[name = tensor("op_1509")]; - tensor var_1510 = mul(x = input_191, y = var_1509)[name = tensor("op_1510")]; - tensor var_1511 = mul(x = x_expanded_25, y = fill_mask_25)[name = tensor("op_1511")]; - tensor filled_window_25 = add(x = var_1510, y = var_1511)[name = tensor("filled_window_25")]; - tensor var_1513 = greater_equal(x = clip_23, y = var_17)[name = tensor("op_1513")]; - tensor var_1514_dtype_0 = const()[name = tensor("op_1514_dtype_0"), val = tensor("fp32")]; - tensor var_1515 = const()[name = tensor("op_1515"), val = tensor([1, 1, 1])]; - tensor var_1514 = cast(dtype = var_1514_dtype_0, x = var_1513)[name = tensor("cast_166")]; - tensor use_roll_25 = reshape(shape = var_1515, x = var_1514)[name = tensor("use_roll_25")]; - tensor var_1517 = sub(x = var_18, y = use_roll_25)[name = tensor("op_1517")]; - tensor var_1518 = mul(x = filled_window_25, y = var_1517)[name = tensor("op_1518")]; - tensor var_1519 = mul(x = rolled_25, y = use_roll_25)[name = tensor("op_1519")]; - tensor input_201 = add(x = var_1518, y = var_1519)[name = tensor("input_201")]; - tensor var_1521 = const()[name = tensor("op_1521"), val = tensor(0x1p+0)]; - tensor var_1522 = add(x = clip_23, y = var_1521)[name = tensor("op_1522")]; - tensor clip_25 = clip(alpha = var_22, beta = var_17, x = var_1522)[name = tensor("clip_25")]; - tensor x_37_axes_0 = const()[name = tensor("x_37_axes_0"), val = tensor([-1])]; - tensor x_37 = layer_norm(axes = x_37_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_201)[name = tensor("x_37")]; - tensor input_203_perm_0 = const()[name = tensor("input_203_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_83_pad_type_0 = const()[name = tensor("inputs_83_pad_type_0"), val = tensor("valid")]; - tensor inputs_83_strides_0 = const()[name = tensor("inputs_83_strides_0"), val = tensor([1])]; - tensor inputs_83_pad_0 = const()[name = tensor("inputs_83_pad_0"), val = tensor([0, 0])]; - tensor inputs_83_dilations_0 = const()[name = tensor("inputs_83_dilations_0"), val = tensor([1])]; - tensor inputs_83_groups_0 = const()[name = tensor("inputs_83_groups_0"), val = tensor(1)]; - tensor input_203 = transpose(perm = input_203_perm_0, x = x_37)[name = tensor("transpose_50")]; - tensor inputs_83 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_83_dilations_0, groups = inputs_83_groups_0, pad = inputs_83_pad_0, pad_type = inputs_83_pad_type_0, strides = inputs_83_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_203)[name = tensor("inputs_83")]; - tensor var_1543_split_sizes_0 = const()[name = tensor("op_1543_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1543_axis_0 = const()[name = tensor("op_1543_axis_0"), val = tensor(1)]; - tensor var_1543_0, tensor var_1543_1 = split(axis = var_1543_axis_0, split_sizes = var_1543_split_sizes_0, x = inputs_83)[name = tensor("op_1543")]; - tensor var_1545 = sigmoid(x = var_1543_1)[name = tensor("op_1545")]; - tensor inputs_85 = mul(x = var_1543_0, y = var_1545)[name = tensor("inputs_85")]; - tensor outputs_aug_25_pad_type_0 = const()[name = tensor("outputs_aug_25_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_25_pad_0 = const()[name = tensor("outputs_aug_25_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_25_groups_0 = const()[name = tensor("outputs_aug_25_groups_0"), val = tensor(256)]; - tensor outputs_aug_25_strides_0 = const()[name = tensor("outputs_aug_25_strides_0"), val = tensor([1])]; - tensor outputs_aug_25_dilations_0 = const()[name = tensor("outputs_aug_25_dilations_0"), val = tensor([1])]; - tensor outputs_aug_25 = conv(dilations = outputs_aug_25_dilations_0, groups = outputs_aug_25_groups_0, pad = outputs_aug_25_pad_0, pad_type = outputs_aug_25_pad_type_0, strides = outputs_aug_25_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_85)[name = tensor("outputs_aug_25")]; - tensor input_205_begin_0 = const()[name = tensor("input_205_begin_0"), val = tensor([0, 0, 0])]; - tensor input_205_end_0 = const()[name = tensor("input_205_end_0"), val = tensor([1, 256, 16])]; - tensor input_205_end_mask_0 = const()[name = tensor("input_205_end_mask_0"), val = tensor([true, true, false])]; - tensor input_205 = slice_by_index(begin = input_205_begin_0, end = input_205_end_0, end_mask = input_205_end_mask_0, x = outputs_aug_25)[name = tensor("input_205")]; - tensor inputs_87 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_205)[name = tensor("inputs_87")]; - tensor input_207 = silu(x = inputs_87)[name = tensor("input_207")]; - tensor input_209_pad_type_0 = const()[name = tensor("input_209_pad_type_0"), val = tensor("valid")]; - tensor input_209_strides_0 = const()[name = tensor("input_209_strides_0"), val = tensor([1])]; - tensor input_209_pad_0 = const()[name = tensor("input_209_pad_0"), val = tensor([0, 0])]; - tensor input_209_dilations_0 = const()[name = tensor("input_209_dilations_0"), val = tensor([1])]; - tensor input_209_groups_0 = const()[name = tensor("input_209_groups_0"), val = tensor(1)]; - tensor input_209 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_209_dilations_0, groups = input_209_groups_0, pad = input_209_pad_0, pad_type = input_209_pad_type_0, strides = input_209_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_207)[name = tensor("input_209")]; - tensor conv_out_25_perm_0 = const()[name = tensor("conv_out_25_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1576_begin_0 = const()[name = tensor("op_1576_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1576_end_0 = const()[name = tensor("op_1576_end_0"), val = tensor([1, 16, 256])]; - tensor var_1576_end_mask_0 = const()[name = tensor("op_1576_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_25 = transpose(perm = conv_out_25_perm_0, x = input_209)[name = tensor("transpose_49")]; - tensor var_1576 = slice_by_index(begin = var_1576_begin_0, end = var_1576_end_0, end_mask = var_1576_end_mask_0, x = conv_out_25)[name = tensor("op_1576")]; - tensor var_1579_begin_0 = const()[name = tensor("op_1579_begin_0"), val = tensor([0, 3, 0])]; - tensor var_1579_end_0 = const()[name = tensor("op_1579_end_0"), val = tensor([1, 4, 256])]; - tensor var_1579_end_mask_0 = const()[name = tensor("op_1579_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1579 = slice_by_index(begin = var_1579_begin_0, end = var_1579_end_0, end_mask = var_1579_end_mask_0, x = x_31)[name = tensor("op_1579")]; - tensor var_1582_begin_0 = const()[name = tensor("op_1582_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1582_end_0 = const()[name = tensor("op_1582_end_0"), val = tensor([1, 16, 256])]; - tensor var_1582_end_mask_0 = const()[name = tensor("op_1582_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1582 = slice_by_index(begin = var_1582_begin_0, end = var_1582_end_0, end_mask = var_1582_end_mask_0, x = input_201)[name = tensor("op_1582")]; + tensor x_expanded_25 = tile(reps = x_expanded_25_reps_0, x = var_957)[name = tensor("x_expanded_25")]; + tensor var_971 = sub(x = var_18, y = fill_mask_25)[name = tensor("op_971")]; + tensor var_972 = mul(x = window_29, y = var_971)[name = tensor("op_972")]; + tensor var_973 = mul(x = x_expanded_25, y = fill_mask_25)[name = tensor("op_973")]; + tensor filled_window_25 = add(x = var_972, y = var_973)[name = tensor("filled_window_25")]; + tensor var_975 = greater_equal(x = clip_23, y = var_17)[name = tensor("op_975")]; + tensor var_976_dtype_0 = const()[name = tensor("op_976_dtype_0"), val = tensor("fp32")]; + tensor var_977 = const()[name = tensor("op_977"), val = tensor([1, 1, 1])]; + tensor var_976 = cast(dtype = var_976_dtype_0, x = var_975)[name = tensor("cast_166")]; + tensor use_roll_25 = reshape(shape = var_977, x = var_976)[name = tensor("use_roll_25")]; + tensor var_979 = sub(x = var_18, y = use_roll_25)[name = tensor("op_979")]; + tensor var_980 = mul(x = filled_window_25, y = var_979)[name = tensor("op_980")]; + tensor var_981 = mul(x = rolled_25, y = use_roll_25)[name = tensor("op_981")]; + tensor window_31 = add(x = var_980, y = var_981)[name = tensor("window_31")]; + tensor var_983 = const()[name = tensor("op_983"), val = tensor(0x1p+0)]; + tensor var_984 = add(x = clip_23, y = var_983)[name = tensor("op_984")]; + tensor clip_25 = clip(alpha = var_22, beta = var_17, x = var_984)[name = tensor("clip_25")]; + tensor var_987_begin_0 = const()[name = tensor("op_987_begin_0"), val = tensor([0, 3, 0])]; + tensor var_987_end_0 = const()[name = tensor("op_987_end_0"), val = tensor([1, 4, 256])]; + tensor var_987_end_mask_0 = const()[name = tensor("op_987_end_mask_0"), val = tensor([true, false, true])]; + tensor var_987 = slice_by_index(begin = var_987_begin_0, end = var_987_end_0, end_mask = var_987_end_mask_0, x = x_15)[name = tensor("op_987")]; + tensor var_990_begin_0 = const()[name = tensor("op_990_begin_0"), val = tensor([0, 1, 0])]; + tensor var_990_end_0 = const()[name = tensor("op_990_end_0"), val = tensor([1, 16, 256])]; + tensor var_990_end_mask_0 = const()[name = tensor("op_990_end_mask_0"), val = tensor([true, true, true])]; + tensor var_990 = slice_by_index(begin = var_990_begin_0, end = var_990_end_0, end_mask = var_990_end_mask_0, x = window_31)[name = tensor("op_990")]; tensor rolled_27_interleave_0 = const()[name = tensor("rolled_27_interleave_0"), val = tensor(false)]; - tensor rolled_27 = concat(axis = var_34, interleave = rolled_27_interleave_0, values = (var_1582, var_1579))[name = tensor("rolled_27")]; + tensor rolled_27 = concat(axis = var_34, interleave = rolled_27_interleave_0, values = (var_990, var_987))[name = tensor("rolled_27")]; tensor clip_26 = clip(alpha = var_22, beta = var_21, x = clip_25)[name = tensor("clip_26")]; - tensor var_1587 = equal(x = encoder__conv_positions, y = clip_26)[name = tensor("op_1587")]; - tensor var_1588_dtype_0 = const()[name = tensor("op_1588_dtype_0"), val = tensor("fp32")]; - tensor var_1589 = const()[name = tensor("op_1589"), val = tensor([1, 16, 1])]; - tensor var_1588 = cast(dtype = var_1588_dtype_0, x = var_1587)[name = tensor("cast_165")]; - tensor fill_mask_27 = reshape(shape = var_1589, x = var_1588)[name = tensor("fill_mask_27")]; + tensor var_995 = equal(x = encoder__conv_positions, y = clip_26)[name = tensor("op_995")]; + tensor var_996_dtype_0 = const()[name = tensor("op_996_dtype_0"), val = tensor("fp32")]; + tensor var_997 = const()[name = tensor("op_997"), val = tensor([1, 16, 1])]; + tensor var_996 = cast(dtype = var_996_dtype_0, x = var_995)[name = tensor("cast_165")]; + tensor fill_mask_27 = reshape(shape = var_997, x = var_996)[name = tensor("fill_mask_27")]; tensor x_expanded_27_reps_0 = const()[name = tensor("x_expanded_27_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_27 = tile(reps = x_expanded_27_reps_0, x = var_1579)[name = tensor("x_expanded_27")]; - tensor var_1593 = sub(x = var_18, y = fill_mask_27)[name = tensor("op_1593")]; - tensor var_1594 = mul(x = input_201, y = var_1593)[name = tensor("op_1594")]; - tensor var_1595 = mul(x = x_expanded_27, y = fill_mask_27)[name = tensor("op_1595")]; - tensor filled_window_27 = add(x = var_1594, y = var_1595)[name = tensor("filled_window_27")]; - tensor var_1597 = greater_equal(x = clip_25, y = var_17)[name = tensor("op_1597")]; - tensor var_1598_dtype_0 = const()[name = tensor("op_1598_dtype_0"), val = tensor("fp32")]; - tensor var_1599 = const()[name = tensor("op_1599"), val = tensor([1, 1, 1])]; - tensor var_1598 = cast(dtype = var_1598_dtype_0, x = var_1597)[name = tensor("cast_164")]; - tensor use_roll_27 = reshape(shape = var_1599, x = var_1598)[name = tensor("use_roll_27")]; - tensor var_1601 = sub(x = var_18, y = use_roll_27)[name = tensor("op_1601")]; - tensor var_1602 = mul(x = filled_window_27, y = var_1601)[name = tensor("op_1602")]; - tensor var_1603 = mul(x = rolled_27, y = use_roll_27)[name = tensor("op_1603")]; - tensor input_211 = add(x = var_1602, y = var_1603)[name = tensor("input_211")]; - tensor var_1605 = const()[name = tensor("op_1605"), val = tensor(0x1p+0)]; - tensor var_1606 = add(x = clip_25, y = var_1605)[name = tensor("op_1606")]; - tensor clip_27 = clip(alpha = var_22, beta = var_17, x = var_1606)[name = tensor("clip_27")]; - tensor x_39_axes_0 = const()[name = tensor("x_39_axes_0"), val = tensor([-1])]; - tensor x_39 = layer_norm(axes = x_39_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_211)[name = tensor("x_39")]; - tensor input_213_perm_0 = const()[name = tensor("input_213_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_89_pad_type_0 = const()[name = tensor("inputs_89_pad_type_0"), val = tensor("valid")]; - tensor inputs_89_strides_0 = const()[name = tensor("inputs_89_strides_0"), val = tensor([1])]; - tensor inputs_89_pad_0 = const()[name = tensor("inputs_89_pad_0"), val = tensor([0, 0])]; - tensor inputs_89_dilations_0 = const()[name = tensor("inputs_89_dilations_0"), val = tensor([1])]; - tensor inputs_89_groups_0 = const()[name = tensor("inputs_89_groups_0"), val = tensor(1)]; - tensor input_213 = transpose(perm = input_213_perm_0, x = x_39)[name = tensor("transpose_48")]; - tensor inputs_89 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_89_dilations_0, groups = inputs_89_groups_0, pad = inputs_89_pad_0, pad_type = inputs_89_pad_type_0, strides = inputs_89_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_213)[name = tensor("inputs_89")]; - tensor var_1627_split_sizes_0 = const()[name = tensor("op_1627_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1627_axis_0 = const()[name = tensor("op_1627_axis_0"), val = tensor(1)]; - tensor var_1627_0, tensor var_1627_1 = split(axis = var_1627_axis_0, split_sizes = var_1627_split_sizes_0, x = inputs_89)[name = tensor("op_1627")]; - tensor var_1629 = sigmoid(x = var_1627_1)[name = tensor("op_1629")]; - tensor inputs_91 = mul(x = var_1627_0, y = var_1629)[name = tensor("inputs_91")]; - tensor outputs_aug_27_pad_type_0 = const()[name = tensor("outputs_aug_27_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_27_pad_0 = const()[name = tensor("outputs_aug_27_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_27_groups_0 = const()[name = tensor("outputs_aug_27_groups_0"), val = tensor(256)]; - tensor outputs_aug_27_strides_0 = const()[name = tensor("outputs_aug_27_strides_0"), val = tensor([1])]; - tensor outputs_aug_27_dilations_0 = const()[name = tensor("outputs_aug_27_dilations_0"), val = tensor([1])]; - tensor outputs_aug_27 = conv(dilations = outputs_aug_27_dilations_0, groups = outputs_aug_27_groups_0, pad = outputs_aug_27_pad_0, pad_type = outputs_aug_27_pad_type_0, strides = outputs_aug_27_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_91)[name = tensor("outputs_aug_27")]; - tensor input_215_begin_0 = const()[name = tensor("input_215_begin_0"), val = tensor([0, 0, 0])]; - tensor input_215_end_0 = const()[name = tensor("input_215_end_0"), val = tensor([1, 256, 16])]; - tensor input_215_end_mask_0 = const()[name = tensor("input_215_end_mask_0"), val = tensor([true, true, false])]; - tensor input_215 = slice_by_index(begin = input_215_begin_0, end = input_215_end_0, end_mask = input_215_end_mask_0, x = outputs_aug_27)[name = tensor("input_215")]; - tensor inputs_93 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_215)[name = tensor("inputs_93")]; - tensor input_217 = silu(x = inputs_93)[name = tensor("input_217")]; - tensor input_219_pad_type_0 = const()[name = tensor("input_219_pad_type_0"), val = tensor("valid")]; - tensor input_219_strides_0 = const()[name = tensor("input_219_strides_0"), val = tensor([1])]; - tensor input_219_pad_0 = const()[name = tensor("input_219_pad_0"), val = tensor([0, 0])]; - tensor input_219_dilations_0 = const()[name = tensor("input_219_dilations_0"), val = tensor([1])]; - tensor input_219_groups_0 = const()[name = tensor("input_219_groups_0"), val = tensor(1)]; - tensor input_219 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_219_dilations_0, groups = input_219_groups_0, pad = input_219_pad_0, pad_type = input_219_pad_type_0, strides = input_219_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_217)[name = tensor("input_219")]; - tensor conv_out_27_perm_0 = const()[name = tensor("conv_out_27_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1660_begin_0 = const()[name = tensor("op_1660_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1660_end_0 = const()[name = tensor("op_1660_end_0"), val = tensor([1, 16, 256])]; - tensor var_1660_end_mask_0 = const()[name = tensor("op_1660_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_27 = transpose(perm = conv_out_27_perm_0, x = input_219)[name = tensor("transpose_47")]; - tensor var_1660 = slice_by_index(begin = var_1660_begin_0, end = var_1660_end_0, end_mask = var_1660_end_mask_0, x = conv_out_27)[name = tensor("op_1660")]; - tensor var_1663_begin_0 = const()[name = tensor("op_1663_begin_0"), val = tensor([0, 4, 0])]; - tensor var_1663_end_0 = const()[name = tensor("op_1663_end_0"), val = tensor([1, 1, 256])]; - tensor var_1663_end_mask_0 = const()[name = tensor("op_1663_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1663 = slice_by_index(begin = var_1663_begin_0, end = var_1663_end_0, end_mask = var_1663_end_mask_0, x = x_31)[name = tensor("op_1663")]; - tensor var_1666_begin_0 = const()[name = tensor("op_1666_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1666_end_0 = const()[name = tensor("op_1666_end_0"), val = tensor([1, 16, 256])]; - tensor var_1666_end_mask_0 = const()[name = tensor("op_1666_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1666 = slice_by_index(begin = var_1666_begin_0, end = var_1666_end_0, end_mask = var_1666_end_mask_0, x = input_211)[name = tensor("op_1666")]; + tensor x_expanded_27 = tile(reps = x_expanded_27_reps_0, x = var_987)[name = tensor("x_expanded_27")]; + tensor var_1001 = sub(x = var_18, y = fill_mask_27)[name = tensor("op_1001")]; + tensor var_1002 = mul(x = window_31, y = var_1001)[name = tensor("op_1002")]; + tensor var_1003 = mul(x = x_expanded_27, y = fill_mask_27)[name = tensor("op_1003")]; + tensor filled_window_27 = add(x = var_1002, y = var_1003)[name = tensor("filled_window_27")]; + tensor var_1005 = greater_equal(x = clip_25, y = var_17)[name = tensor("op_1005")]; + tensor var_1006_dtype_0 = const()[name = tensor("op_1006_dtype_0"), val = tensor("fp32")]; + tensor var_1007 = const()[name = tensor("op_1007"), val = tensor([1, 1, 1])]; + tensor var_1006 = cast(dtype = var_1006_dtype_0, x = var_1005)[name = tensor("cast_164")]; + tensor use_roll_27 = reshape(shape = var_1007, x = var_1006)[name = tensor("use_roll_27")]; + tensor var_1009 = sub(x = var_18, y = use_roll_27)[name = tensor("op_1009")]; + tensor var_1010 = mul(x = filled_window_27, y = var_1009)[name = tensor("op_1010")]; + tensor var_1011 = mul(x = rolled_27, y = use_roll_27)[name = tensor("op_1011")]; + tensor window_33 = add(x = var_1010, y = var_1011)[name = tensor("window_33")]; + tensor var_1013 = const()[name = tensor("op_1013"), val = tensor(0x1p+0)]; + tensor var_1014 = add(x = clip_25, y = var_1013)[name = tensor("op_1014")]; + tensor clip_27 = clip(alpha = var_22, beta = var_17, x = var_1014)[name = tensor("clip_27")]; + tensor var_1017_begin_0 = const()[name = tensor("op_1017_begin_0"), val = tensor([0, 4, 0])]; + tensor var_1017_end_0 = const()[name = tensor("op_1017_end_0"), val = tensor([1, 1, 256])]; + tensor var_1017_end_mask_0 = const()[name = tensor("op_1017_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1017 = slice_by_index(begin = var_1017_begin_0, end = var_1017_end_0, end_mask = var_1017_end_mask_0, x = x_15)[name = tensor("op_1017")]; + tensor var_1020_begin_0 = const()[name = tensor("op_1020_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1020_end_0 = const()[name = tensor("op_1020_end_0"), val = tensor([1, 16, 256])]; + tensor var_1020_end_mask_0 = const()[name = tensor("op_1020_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1020 = slice_by_index(begin = var_1020_begin_0, end = var_1020_end_0, end_mask = var_1020_end_mask_0, x = window_33)[name = tensor("op_1020")]; tensor rolled_29_interleave_0 = const()[name = tensor("rolled_29_interleave_0"), val = tensor(false)]; - tensor rolled_29 = concat(axis = var_34, interleave = rolled_29_interleave_0, values = (var_1666, var_1663))[name = tensor("rolled_29")]; + tensor rolled_29 = concat(axis = var_34, interleave = rolled_29_interleave_0, values = (var_1020, var_1017))[name = tensor("rolled_29")]; tensor clip_28 = clip(alpha = var_22, beta = var_21, x = clip_27)[name = tensor("clip_28")]; - tensor var_1671 = equal(x = encoder__conv_positions, y = clip_28)[name = tensor("op_1671")]; - tensor var_1672_dtype_0 = const()[name = tensor("op_1672_dtype_0"), val = tensor("fp32")]; - tensor var_1673 = const()[name = tensor("op_1673"), val = tensor([1, 16, 1])]; - tensor var_1672 = cast(dtype = var_1672_dtype_0, x = var_1671)[name = tensor("cast_163")]; - tensor fill_mask_29 = reshape(shape = var_1673, x = var_1672)[name = tensor("fill_mask_29")]; + tensor var_1025 = equal(x = encoder__conv_positions, y = clip_28)[name = tensor("op_1025")]; + tensor var_1026_dtype_0 = const()[name = tensor("op_1026_dtype_0"), val = tensor("fp32")]; + tensor var_1027 = const()[name = tensor("op_1027"), val = tensor([1, 16, 1])]; + tensor var_1026 = cast(dtype = var_1026_dtype_0, x = var_1025)[name = tensor("cast_163")]; + tensor fill_mask_29 = reshape(shape = var_1027, x = var_1026)[name = tensor("fill_mask_29")]; tensor x_expanded_29_reps_0 = const()[name = tensor("x_expanded_29_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_29 = tile(reps = x_expanded_29_reps_0, x = var_1663)[name = tensor("x_expanded_29")]; - tensor var_1677 = sub(x = var_18, y = fill_mask_29)[name = tensor("op_1677")]; - tensor var_1678 = mul(x = input_211, y = var_1677)[name = tensor("op_1678")]; - tensor var_1679 = mul(x = x_expanded_29, y = fill_mask_29)[name = tensor("op_1679")]; - tensor filled_window_29 = add(x = var_1678, y = var_1679)[name = tensor("filled_window_29")]; - tensor var_1681 = greater_equal(x = clip_27, y = var_17)[name = tensor("op_1681")]; - tensor var_1682_dtype_0 = const()[name = tensor("op_1682_dtype_0"), val = tensor("fp32")]; - tensor var_1683 = const()[name = tensor("op_1683"), val = tensor([1, 1, 1])]; - tensor var_1682 = cast(dtype = var_1682_dtype_0, x = var_1681)[name = tensor("cast_162")]; - tensor use_roll_29 = reshape(shape = var_1683, x = var_1682)[name = tensor("use_roll_29")]; - tensor var_1685 = sub(x = var_18, y = use_roll_29)[name = tensor("op_1685")]; - tensor var_1686 = mul(x = filled_window_29, y = var_1685)[name = tensor("op_1686")]; - tensor var_1687 = mul(x = rolled_29, y = use_roll_29)[name = tensor("op_1687")]; - tensor input_221 = add(x = var_1686, y = var_1687)[name = tensor("input_221")]; - tensor var_1689 = const()[name = tensor("op_1689"), val = tensor(0x1p+0)]; - tensor var_1690 = add(x = clip_27, y = var_1689)[name = tensor("op_1690")]; - tensor clip_29 = clip(alpha = var_22, beta = var_17, x = var_1690)[name = tensor("clip_29")]; - tensor x_41_axes_0 = const()[name = tensor("x_41_axes_0"), val = tensor([-1])]; - tensor x_41 = layer_norm(axes = x_41_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_221)[name = tensor("x_41")]; - tensor input_223_perm_0 = const()[name = tensor("input_223_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_95_pad_type_0 = const()[name = tensor("inputs_95_pad_type_0"), val = tensor("valid")]; - tensor inputs_95_strides_0 = const()[name = tensor("inputs_95_strides_0"), val = tensor([1])]; - tensor inputs_95_pad_0 = const()[name = tensor("inputs_95_pad_0"), val = tensor([0, 0])]; - tensor inputs_95_dilations_0 = const()[name = tensor("inputs_95_dilations_0"), val = tensor([1])]; - tensor inputs_95_groups_0 = const()[name = tensor("inputs_95_groups_0"), val = tensor(1)]; - tensor input_223 = transpose(perm = input_223_perm_0, x = x_41)[name = tensor("transpose_46")]; - tensor inputs_95 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_95_dilations_0, groups = inputs_95_groups_0, pad = inputs_95_pad_0, pad_type = inputs_95_pad_type_0, strides = inputs_95_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_223)[name = tensor("inputs_95")]; - tensor var_1711_split_sizes_0 = const()[name = tensor("op_1711_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1711_axis_0 = const()[name = tensor("op_1711_axis_0"), val = tensor(1)]; - tensor var_1711_0, tensor var_1711_1 = split(axis = var_1711_axis_0, split_sizes = var_1711_split_sizes_0, x = inputs_95)[name = tensor("op_1711")]; - tensor var_1713 = sigmoid(x = var_1711_1)[name = tensor("op_1713")]; - tensor inputs_97 = mul(x = var_1711_0, y = var_1713)[name = tensor("inputs_97")]; - tensor outputs_aug_29_pad_type_0 = const()[name = tensor("outputs_aug_29_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_29_pad_0 = const()[name = tensor("outputs_aug_29_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_29_groups_0 = const()[name = tensor("outputs_aug_29_groups_0"), val = tensor(256)]; - tensor outputs_aug_29_strides_0 = const()[name = tensor("outputs_aug_29_strides_0"), val = tensor([1])]; - tensor outputs_aug_29_dilations_0 = const()[name = tensor("outputs_aug_29_dilations_0"), val = tensor([1])]; - tensor outputs_aug_29 = conv(dilations = outputs_aug_29_dilations_0, groups = outputs_aug_29_groups_0, pad = outputs_aug_29_pad_0, pad_type = outputs_aug_29_pad_type_0, strides = outputs_aug_29_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_97)[name = tensor("outputs_aug_29")]; - tensor input_225_begin_0 = const()[name = tensor("input_225_begin_0"), val = tensor([0, 0, 0])]; - tensor input_225_end_0 = const()[name = tensor("input_225_end_0"), val = tensor([1, 256, 16])]; - tensor input_225_end_mask_0 = const()[name = tensor("input_225_end_mask_0"), val = tensor([true, true, false])]; - tensor input_225 = slice_by_index(begin = input_225_begin_0, end = input_225_end_0, end_mask = input_225_end_mask_0, x = outputs_aug_29)[name = tensor("input_225")]; - tensor inputs_99 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_225)[name = tensor("inputs_99")]; - tensor input_227 = silu(x = inputs_99)[name = tensor("input_227")]; - tensor input_229_pad_type_0 = const()[name = tensor("input_229_pad_type_0"), val = tensor("valid")]; - tensor input_229_strides_0 = const()[name = tensor("input_229_strides_0"), val = tensor([1])]; - tensor input_229_pad_0 = const()[name = tensor("input_229_pad_0"), val = tensor([0, 0])]; - tensor input_229_dilations_0 = const()[name = tensor("input_229_dilations_0"), val = tensor([1])]; - tensor input_229_groups_0 = const()[name = tensor("input_229_groups_0"), val = tensor(1)]; - tensor input_229 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_229_dilations_0, groups = input_229_groups_0, pad = input_229_pad_0, pad_type = input_229_pad_type_0, strides = input_229_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_227)[name = tensor("input_229")]; - tensor conv_out_29_perm_0 = const()[name = tensor("conv_out_29_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1744_begin_0 = const()[name = tensor("op_1744_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1744_end_0 = const()[name = tensor("op_1744_end_0"), val = tensor([1, 16, 256])]; - tensor var_1744_end_mask_0 = const()[name = tensor("op_1744_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_29 = transpose(perm = conv_out_29_perm_0, x = input_229)[name = tensor("transpose_45")]; - tensor var_1744 = slice_by_index(begin = var_1744_begin_0, end = var_1744_end_0, end_mask = var_1744_end_mask_0, x = conv_out_29)[name = tensor("op_1744")]; - tensor var_1747_interleave_0 = const()[name = tensor("op_1747_interleave_0"), val = tensor(false)]; - tensor var_1747 = concat(axis = var_34, interleave = var_1747_interleave_0, values = (var_1408, var_1492, var_1576, var_1660, var_1744))[name = tensor("op_1747")]; - tensor input_231 = add(x = x_31, y = var_1747)[name = tensor("input_231")]; - tensor input_233_axes_0 = const()[name = tensor("input_233_axes_0"), val = tensor([-1])]; - tensor input_233 = layer_norm(axes = input_233_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_231)[name = tensor("input_233")]; - tensor inputs_101 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_233)[name = tensor("linear_26")]; - tensor input_235 = silu(x = inputs_101)[name = tensor("input_235")]; - tensor input_239 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_235)[name = tensor("linear_27")]; - tensor var_1770 = const()[name = tensor("op_1770"), val = tensor(0x1p-1)]; - tensor var_1771 = mul(x = input_239, y = var_1770)[name = tensor("op_1771")]; - tensor input_241 = add(x = var_1771, y = input_231)[name = tensor("input_241")]; - tensor input_243_axes_0 = const()[name = tensor("input_243_axes_0"), val = tensor([-1])]; - tensor input_243 = layer_norm(axes = input_243_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_36, gamma = encoder_layer_norm_2_weight, x = input_241)[name = tensor("input_243")]; - tensor input_245_axes_0 = const()[name = tensor("input_245_axes_0"), val = tensor([-1])]; - tensor input_245 = layer_norm(axes = input_245_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_243)[name = tensor("input_245")]; - tensor inputs_103 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_245)[name = tensor("linear_28")]; - tensor input_247 = silu(x = inputs_103)[name = tensor("input_247")]; - tensor input_251 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_247)[name = tensor("linear_29")]; - tensor var_1800 = const()[name = tensor("op_1800"), val = tensor(0x1p-1)]; - tensor var_1801 = mul(x = input_251, y = var_1800)[name = tensor("op_1801")]; - tensor input_253 = add(x = var_1801, y = input_243)[name = tensor("input_253")]; - tensor x_43_axes_0 = const()[name = tensor("x_43_axes_0"), val = tensor([-1])]; - tensor x_43 = layer_norm(axes = x_43_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_36, gamma = encoder_ret_lns_3_weight, x = input_253)[name = tensor("x_43")]; + tensor x_expanded_29 = tile(reps = x_expanded_29_reps_0, x = var_1017)[name = tensor("x_expanded_29")]; + tensor var_1031 = sub(x = var_18, y = fill_mask_29)[name = tensor("op_1031")]; + tensor var_1032 = mul(x = window_33, y = var_1031)[name = tensor("op_1032")]; + tensor var_1033 = mul(x = x_expanded_29, y = fill_mask_29)[name = tensor("op_1033")]; + tensor filled_window_29 = add(x = var_1032, y = var_1033)[name = tensor("filled_window_29")]; + tensor var_1035 = greater_equal(x = clip_27, y = var_17)[name = tensor("op_1035")]; + tensor var_1036_dtype_0 = const()[name = tensor("op_1036_dtype_0"), val = tensor("fp32")]; + tensor var_1037 = const()[name = tensor("op_1037"), val = tensor([1, 1, 1])]; + tensor var_1036 = cast(dtype = var_1036_dtype_0, x = var_1035)[name = tensor("cast_162")]; + tensor use_roll_29 = reshape(shape = var_1037, x = var_1036)[name = tensor("use_roll_29")]; + tensor var_1039 = sub(x = var_18, y = use_roll_29)[name = tensor("op_1039")]; + tensor var_1040 = mul(x = filled_window_29, y = var_1039)[name = tensor("op_1040")]; + tensor var_1041 = mul(x = rolled_29, y = use_roll_29)[name = tensor("op_1041")]; + tensor window_35 = add(x = var_1040, y = var_1041)[name = tensor("window_35")]; + tensor var_1043 = const()[name = tensor("op_1043"), val = tensor(0x1p+0)]; + tensor var_1044 = add(x = clip_27, y = var_1043)[name = tensor("op_1044")]; + tensor clip_29 = clip(alpha = var_22, beta = var_17, x = var_1044)[name = tensor("clip_29")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_31, interleave = input_101_interleave_0, values = (window_27, window_29, window_31, window_33, window_35))[name = tensor("input_101")]; + tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; + tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; + tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_23_pad_type_0 = const()[name = tensor("inputs_23_pad_type_0"), val = tensor("valid")]; + tensor inputs_23_strides_0 = const()[name = tensor("inputs_23_strides_0"), val = tensor([1])]; + tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; + tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; + tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; + tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; + tensor var_1067_split_sizes_0 = const()[name = tensor("op_1067_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1067_axis_0 = const()[name = tensor("op_1067_axis_0"), val = tensor(1)]; + tensor var_1067_0, tensor var_1067_1 = split(axis = var_1067_axis_0, split_sizes = var_1067_split_sizes_0, x = inputs_23)[name = tensor("op_1067")]; + tensor var_1069 = sigmoid(x = var_1067_1)[name = tensor("op_1069")]; + tensor inputs_25 = mul(x = var_1067_0, y = var_1069)[name = tensor("inputs_25")]; + tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; + tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; + tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; + tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_25)[name = tensor("outputs_aug_5")]; + tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; + tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([5, 256, 16])]; + tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; + tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_5)[name = tensor("input_105")]; + tensor inputs_27 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_105)[name = tensor("inputs_27")]; + tensor input_107 = silu(x = inputs_27)[name = tensor("input_107")]; + tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; + tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; + tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; + tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; + tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; + tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; + tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1100_begin_0 = const()[name = tensor("op_1100_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1100_end_0 = const()[name = tensor("op_1100_end_0"), val = tensor([5, 16, 256])]; + tensor var_1100_end_mask_0 = const()[name = tensor("op_1100_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_1100 = slice_by_index(begin = var_1100_begin_0, end = var_1100_end_0, end_mask = var_1100_end_mask_0, x = conv_out_5)[name = tensor("op_1100")]; + tensor var_1102_perm_0 = const()[name = tensor("op_1102_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1102 = transpose(perm = var_1102_perm_0, x = var_1100)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_1102)[name = tensor("input_111")]; + tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; + tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; + tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; + tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; + tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; + tensor var_1125 = const()[name = tensor("op_1125"), val = tensor(0x1p-1)]; + tensor var_1126 = mul(x = input_119, y = var_1125)[name = tensor("op_1126")]; + tensor input_121 = add(x = var_1126, y = input_111)[name = tensor("input_121")]; + tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; + tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_36, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; + tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; + tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; + tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; + tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; + tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; + tensor var_1155 = const()[name = tensor("op_1155"), val = tensor(0x1p-1)]; + tensor var_1156 = mul(x = input_131, y = var_1155)[name = tensor("op_1156")]; + tensor input_133 = add(x = var_1156, y = input_123)[name = tensor("input_133")]; + tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; + tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_36, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; tensor prev_kv_7_end_0 = const()[name = tensor("prev_kv_7_end_0"), val = tensor([4, 1, 4, 64, 64])]; tensor prev_kv_7_end_mask_0 = const()[name = tensor("prev_kv_7_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1641,455 +1180,301 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_1815 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_43)[name = tensor("linear_30")]; - tensor var_1816 = const()[name = tensor("op_1816"), val = tensor([1, 5, 4, 64])]; - tensor var_1817 = reshape(shape = var_1816, x = var_1815)[name = tensor("op_1817")]; + tensor var_1170 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_1171 = const()[name = tensor("op_1171"), val = tensor([1, 5, 4, 64])]; + tensor var_1172 = reshape(shape = var_1171, x = var_1170)[name = tensor("op_1172")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1821 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_43)[name = tensor("linear_31")]; - tensor var_1822 = const()[name = tensor("op_1822"), val = tensor(0x1p-3)]; - tensor var_1823 = mul(x = var_1821, y = var_1822)[name = tensor("op_1823")]; - tensor var_1824 = const()[name = tensor("op_1824"), val = tensor([1, 5, 4, 64])]; - tensor var_1825 = reshape(shape = var_1824, x = var_1823)[name = tensor("op_1825")]; + tensor var_1176 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_1177 = const()[name = tensor("op_1177"), val = tensor(0x1p-3)]; + tensor var_1178 = mul(x = var_1176, y = var_1177)[name = tensor("op_1178")]; + tensor var_1179 = const()[name = tensor("op_1179"), val = tensor([1, 5, 4, 64])]; + tensor var_1180 = reshape(shape = var_1179, x = var_1178)[name = tensor("op_1180")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1829 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_43)[name = tensor("linear_32")]; - tensor var_1830 = const()[name = tensor("op_1830"), val = tensor([1, 5, 4, 64])]; - tensor var_1831 = reshape(shape = var_1830, x = var_1829)[name = tensor("op_1831")]; + tensor var_1184 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_1185 = const()[name = tensor("op_1185"), val = tensor([1, 5, 4, 64])]; + tensor var_1186 = reshape(shape = var_1185, x = var_1184)[name = tensor("op_1186")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_257 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_43)[name = tensor("linear_33")]; + tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; tensor s_t_7 = add(x = prev_scale_7, y = encoder__t_index)[name = tensor("s_t_7")]; tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_1825)[name = tensor("transpose_43")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_1817)[name = tensor("transpose_44")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_1180)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_1172)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_1841 = const()[name = tensor("op_1841"), val = tensor([5, 1])]; - tensor var_1842 = reshape(shape = var_1841, x = sqrt_s_t_7)[name = tensor("op_1842")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_1842)[name = tensor("M_7")]; - tensor var_1844 = mul(x = qk_7, y = M_7)[name = tensor("op_1844")]; + tensor var_1196 = const()[name = tensor("op_1196"), val = tensor([5, 1])]; + tensor var_1197 = reshape(shape = var_1196, x = sqrt_s_t_7)[name = tensor("op_1197")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_1197)[name = tensor("M_7")]; + tensor var_1199 = mul(x = qk_7, y = M_7)[name = tensor("op_1199")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_1831)[name = tensor("transpose_42")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1844, y = v_7)[name = tensor("inner_7")]; - tensor var_1846_transpose_x_0 = const()[name = tensor("op_1846_transpose_x_0"), val = tensor(false)]; - tensor var_1846_transpose_y_0 = const()[name = tensor("op_1846_transpose_y_0"), val = tensor(false)]; - tensor var_1846 = matmul(transpose_x = var_1846_transpose_x_0, transpose_y = var_1846_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1846")]; - tensor var_1847 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1847")]; - tensor var_1848 = const()[name = tensor("op_1848"), val = tensor([1, 1, 5, 1])]; - tensor var_1849 = reshape(shape = var_1848, x = var_1847)[name = tensor("op_1849")]; - tensor cross_7 = mul(x = var_1846, y = var_1849)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_1186)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1199, y = v_7)[name = tensor("inner_7")]; + tensor var_1201_transpose_x_0 = const()[name = tensor("op_1201_transpose_x_0"), val = tensor(false)]; + tensor var_1201_transpose_y_0 = const()[name = tensor("op_1201_transpose_y_0"), val = tensor(false)]; + tensor var_1201 = matmul(transpose_x = var_1201_transpose_x_0, transpose_y = var_1201_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1201")]; + tensor var_1202 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1202")]; + tensor var_1203 = const()[name = tensor("op_1203"), val = tensor([1, 1, 5, 1])]; + tensor var_1204 = reshape(shape = var_1203, x = var_1202)[name = tensor("op_1204")]; + tensor cross_7 = mul(x = var_1201, y = var_1204)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_1852 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1852")]; - tensor var_1854_transpose_x_1 = const()[name = tensor("op_1854_transpose_x_1"), val = tensor(true)]; - tensor var_1854_transpose_y_1 = const()[name = tensor("op_1854_transpose_y_1"), val = tensor(false)]; - tensor var_1854 = matmul(transpose_x = var_1854_transpose_x_1, transpose_y = var_1854_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1854")]; - tensor new_kv_unnorm_7 = add(x = var_1852, y = var_1854)[name = tensor("new_kv_unnorm_7")]; - tensor var_1856 = const()[name = tensor("op_1856"), val = tensor(0x1.4p+2)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_1856)[name = tensor("new_scale_7")]; - tensor var_1858 = sqrt(x = new_scale_7)[name = tensor("op_1858")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1858)[name = tensor("nkv_1")]; - tensor var_1860_perm_0 = const()[name = tensor("op_1860_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1207 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1207")]; + tensor var_1209_transpose_x_1 = const()[name = tensor("op_1209_transpose_x_1"), val = tensor(true)]; + tensor var_1209_transpose_y_1 = const()[name = tensor("op_1209_transpose_y_1"), val = tensor(false)]; + tensor var_1209 = matmul(transpose_x = var_1209_transpose_x_1, transpose_y = var_1209_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1209")]; + tensor new_kv_unnorm_7 = add(x = var_1207, y = var_1209)[name = tensor("new_kv_unnorm_7")]; + tensor var_1211 = const()[name = tensor("op_1211"), val = tensor(0x1.4p+2)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_1211)[name = tensor("new_scale_7")]; + tensor var_1213 = sqrt(x = new_scale_7)[name = tensor("op_1213")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1213)[name = tensor("nkv_1")]; + tensor var_1215_perm_0 = const()[name = tensor("op_1215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_1860 = transpose(perm = var_1860_perm_0, x = out_19)[name = tensor("transpose_41")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_25, x = var_1860)[name = tensor("out_21")]; - tensor var_1864 = const()[name = tensor("op_1864"), val = tensor([1, 5, 256])]; - tensor out_23 = reshape(shape = var_1864, x = out_21)[name = tensor("out_23")]; - tensor var_1866 = silu(x = input_257)[name = tensor("op_1866")]; - tensor input_259 = mul(x = var_1866, y = out_23)[name = tensor("input_259")]; - tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_259)[name = tensor("linear_34")]; - tensor x_45 = add(x = input_253, y = ret_out_7)[name = tensor("x_45")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor var_1215 = transpose(perm = var_1215_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_25, x = var_1215)[name = tensor("out_21")]; + tensor var_1219 = const()[name = tensor("op_1219"), val = tensor([1, 5, 256])]; + tensor out_23 = reshape(shape = var_1219, x = out_21)[name = tensor("out_23")]; + tensor var_1221 = silu(x = input_137)[name = tensor("op_1221")]; + tensor input_139 = mul(x = var_1221, y = out_23)[name = tensor("input_139")]; + tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; + tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; + tensor window_37_begin_0 = const()[name = tensor("window_37_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_37_end_0 = const()[name = tensor("window_37_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_37_end_mask_0 = const()[name = tensor("window_37_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_37_squeeze_mask_0 = const()[name = tensor("window_37_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_37 = slice_by_index(begin = window_37_begin_0, end = window_37_end_0, end_mask = window_37_end_mask_0, squeeze_mask = window_37_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_37")]; tensor filled_31_begin_0 = const()[name = tensor("filled_31_begin_0"), val = tensor([3, 0])]; tensor filled_31_end_0 = const()[name = tensor("filled_31_end_0"), val = tensor([4, 1])]; tensor filled_31_end_mask_0 = const()[name = tensor("filled_31_end_mask_0"), val = tensor([false, true])]; tensor filled_31_squeeze_mask_0 = const()[name = tensor("filled_31_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_31 = slice_by_index(begin = filled_31_begin_0, end = filled_31_end_0, end_mask = filled_31_end_mask_0, squeeze_mask = filled_31_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_31")]; - tensor var_1875_begin_0 = const()[name = tensor("op_1875_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1875_end_0 = const()[name = tensor("op_1875_end_0"), val = tensor([1, 1, 256])]; - tensor var_1875_end_mask_0 = const()[name = tensor("op_1875_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1875 = slice_by_index(begin = var_1875_begin_0, end = var_1875_end_0, end_mask = var_1875_end_mask_0, x = x_45)[name = tensor("op_1875")]; - tensor var_1878_begin_0 = const()[name = tensor("op_1878_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1878_end_0 = const()[name = tensor("op_1878_end_0"), val = tensor([1, 16, 256])]; - tensor var_1878_end_mask_0 = const()[name = tensor("op_1878_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1878 = slice_by_index(begin = var_1878_begin_0, end = var_1878_end_0, end_mask = var_1878_end_mask_0, x = window)[name = tensor("op_1878")]; + tensor var_1230_begin_0 = const()[name = tensor("op_1230_begin_0"), val = tensor([0, 0, 0])]; + tensor var_1230_end_0 = const()[name = tensor("op_1230_end_0"), val = tensor([1, 1, 256])]; + tensor var_1230_end_mask_0 = const()[name = tensor("op_1230_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1230 = slice_by_index(begin = var_1230_begin_0, end = var_1230_end_0, end_mask = var_1230_end_mask_0, x = x_21)[name = tensor("op_1230")]; + tensor var_1233_begin_0 = const()[name = tensor("op_1233_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1233_end_0 = const()[name = tensor("op_1233_end_0"), val = tensor([1, 16, 256])]; + tensor var_1233_end_mask_0 = const()[name = tensor("op_1233_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1233 = slice_by_index(begin = var_1233_begin_0, end = var_1233_end_0, end_mask = var_1233_end_mask_0, x = window_37)[name = tensor("op_1233")]; tensor rolled_31_interleave_0 = const()[name = tensor("rolled_31_interleave_0"), val = tensor(false)]; - tensor rolled_31 = concat(axis = var_34, interleave = rolled_31_interleave_0, values = (var_1878, var_1875))[name = tensor("rolled_31")]; + tensor rolled_31 = concat(axis = var_34, interleave = rolled_31_interleave_0, values = (var_1233, var_1230))[name = tensor("rolled_31")]; tensor clip_30 = clip(alpha = var_22, beta = var_21, x = filled_31)[name = tensor("clip_30")]; - tensor var_1883 = equal(x = encoder__conv_positions, y = clip_30)[name = tensor("op_1883")]; - tensor var_1884_dtype_0 = const()[name = tensor("op_1884_dtype_0"), val = tensor("fp32")]; - tensor var_1885 = const()[name = tensor("op_1885"), val = tensor([1, 16, 1])]; - tensor var_1884 = cast(dtype = var_1884_dtype_0, x = var_1883)[name = tensor("cast_161")]; - tensor fill_mask_31 = reshape(shape = var_1885, x = var_1884)[name = tensor("fill_mask_31")]; + tensor var_1238 = equal(x = encoder__conv_positions, y = clip_30)[name = tensor("op_1238")]; + tensor var_1239_dtype_0 = const()[name = tensor("op_1239_dtype_0"), val = tensor("fp32")]; + tensor var_1240 = const()[name = tensor("op_1240"), val = tensor([1, 16, 1])]; + tensor var_1239 = cast(dtype = var_1239_dtype_0, x = var_1238)[name = tensor("cast_161")]; + tensor fill_mask_31 = reshape(shape = var_1240, x = var_1239)[name = tensor("fill_mask_31")]; tensor x_expanded_31_reps_0 = const()[name = tensor("x_expanded_31_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_31 = tile(reps = x_expanded_31_reps_0, x = var_1875)[name = tensor("x_expanded_31")]; - tensor var_1889 = sub(x = var_18, y = fill_mask_31)[name = tensor("op_1889")]; - tensor var_1890 = mul(x = window, y = var_1889)[name = tensor("op_1890")]; - tensor var_1891 = mul(x = x_expanded_31, y = fill_mask_31)[name = tensor("op_1891")]; - tensor filled_window_31 = add(x = var_1890, y = var_1891)[name = tensor("filled_window_31")]; - tensor var_1893 = greater_equal(x = filled_31, y = var_17)[name = tensor("op_1893")]; - tensor var_1894_dtype_0 = const()[name = tensor("op_1894_dtype_0"), val = tensor("fp32")]; - tensor var_1895 = const()[name = tensor("op_1895"), val = tensor([1, 1, 1])]; - tensor var_1894 = cast(dtype = var_1894_dtype_0, x = var_1893)[name = tensor("cast_160")]; - tensor use_roll_31 = reshape(shape = var_1895, x = var_1894)[name = tensor("use_roll_31")]; - tensor var_1897 = sub(x = var_18, y = use_roll_31)[name = tensor("op_1897")]; - tensor var_1898 = mul(x = filled_window_31, y = var_1897)[name = tensor("op_1898")]; - tensor var_1899 = mul(x = rolled_31, y = use_roll_31)[name = tensor("op_1899")]; - tensor input_261 = add(x = var_1898, y = var_1899)[name = tensor("input_261")]; - tensor var_1901 = const()[name = tensor("op_1901"), val = tensor(0x1p+0)]; - tensor var_1902 = add(x = filled_31, y = var_1901)[name = tensor("op_1902")]; - tensor clip_31 = clip(alpha = var_22, beta = var_17, x = var_1902)[name = tensor("clip_31")]; - tensor x_47_axes_0 = const()[name = tensor("x_47_axes_0"), val = tensor([-1])]; - tensor x_47 = layer_norm(axes = x_47_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_261)[name = tensor("x_47")]; - tensor input_263_perm_0 = const()[name = tensor("input_263_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_105_pad_type_0 = const()[name = tensor("inputs_105_pad_type_0"), val = tensor("valid")]; - tensor inputs_105_strides_0 = const()[name = tensor("inputs_105_strides_0"), val = tensor([1])]; - tensor inputs_105_pad_0 = const()[name = tensor("inputs_105_pad_0"), val = tensor([0, 0])]; - tensor inputs_105_dilations_0 = const()[name = tensor("inputs_105_dilations_0"), val = tensor([1])]; - tensor inputs_105_groups_0 = const()[name = tensor("inputs_105_groups_0"), val = tensor(1)]; - tensor input_263 = transpose(perm = input_263_perm_0, x = x_47)[name = tensor("transpose_40")]; - tensor inputs_105 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_105_dilations_0, groups = inputs_105_groups_0, pad = inputs_105_pad_0, pad_type = inputs_105_pad_type_0, strides = inputs_105_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_263)[name = tensor("inputs_105")]; - tensor var_1923_split_sizes_0 = const()[name = tensor("op_1923_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1923_axis_0 = const()[name = tensor("op_1923_axis_0"), val = tensor(1)]; - tensor var_1923_0, tensor var_1923_1 = split(axis = var_1923_axis_0, split_sizes = var_1923_split_sizes_0, x = inputs_105)[name = tensor("op_1923")]; - tensor var_1925 = sigmoid(x = var_1923_1)[name = tensor("op_1925")]; - tensor inputs_107 = mul(x = var_1923_0, y = var_1925)[name = tensor("inputs_107")]; - tensor outputs_aug_31_pad_type_0 = const()[name = tensor("outputs_aug_31_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_31_pad_0 = const()[name = tensor("outputs_aug_31_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_31_groups_0 = const()[name = tensor("outputs_aug_31_groups_0"), val = tensor(256)]; - tensor outputs_aug_31_strides_0 = const()[name = tensor("outputs_aug_31_strides_0"), val = tensor([1])]; - tensor outputs_aug_31_dilations_0 = const()[name = tensor("outputs_aug_31_dilations_0"), val = tensor([1])]; - tensor outputs_aug_31 = conv(dilations = outputs_aug_31_dilations_0, groups = outputs_aug_31_groups_0, pad = outputs_aug_31_pad_0, pad_type = outputs_aug_31_pad_type_0, strides = outputs_aug_31_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_107)[name = tensor("outputs_aug_31")]; - tensor input_265_begin_0 = const()[name = tensor("input_265_begin_0"), val = tensor([0, 0, 0])]; - tensor input_265_end_0 = const()[name = tensor("input_265_end_0"), val = tensor([1, 256, 16])]; - tensor input_265_end_mask_0 = const()[name = tensor("input_265_end_mask_0"), val = tensor([true, true, false])]; - tensor input_265 = slice_by_index(begin = input_265_begin_0, end = input_265_end_0, end_mask = input_265_end_mask_0, x = outputs_aug_31)[name = tensor("input_265")]; - tensor inputs_109 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_265)[name = tensor("inputs_109")]; - tensor input_267 = silu(x = inputs_109)[name = tensor("input_267")]; - tensor input_269_pad_type_0 = const()[name = tensor("input_269_pad_type_0"), val = tensor("valid")]; - tensor input_269_strides_0 = const()[name = tensor("input_269_strides_0"), val = tensor([1])]; - tensor input_269_pad_0 = const()[name = tensor("input_269_pad_0"), val = tensor([0, 0])]; - tensor input_269_dilations_0 = const()[name = tensor("input_269_dilations_0"), val = tensor([1])]; - tensor input_269_groups_0 = const()[name = tensor("input_269_groups_0"), val = tensor(1)]; - tensor input_269 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_269_dilations_0, groups = input_269_groups_0, pad = input_269_pad_0, pad_type = input_269_pad_type_0, strides = input_269_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_267)[name = tensor("input_269")]; - tensor conv_out_31_perm_0 = const()[name = tensor("conv_out_31_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1956_begin_0 = const()[name = tensor("op_1956_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1956_end_0 = const()[name = tensor("op_1956_end_0"), val = tensor([1, 16, 256])]; - tensor var_1956_end_mask_0 = const()[name = tensor("op_1956_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_31 = transpose(perm = conv_out_31_perm_0, x = input_269)[name = tensor("transpose_39")]; - tensor var_1956 = slice_by_index(begin = var_1956_begin_0, end = var_1956_end_0, end_mask = var_1956_end_mask_0, x = conv_out_31)[name = tensor("op_1956")]; - tensor var_1959_begin_0 = const()[name = tensor("op_1959_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1959_end_0 = const()[name = tensor("op_1959_end_0"), val = tensor([1, 2, 256])]; - tensor var_1959_end_mask_0 = const()[name = tensor("op_1959_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1959 = slice_by_index(begin = var_1959_begin_0, end = var_1959_end_0, end_mask = var_1959_end_mask_0, x = x_45)[name = tensor("op_1959")]; - tensor var_1962_begin_0 = const()[name = tensor("op_1962_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1962_end_0 = const()[name = tensor("op_1962_end_0"), val = tensor([1, 16, 256])]; - tensor var_1962_end_mask_0 = const()[name = tensor("op_1962_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1962 = slice_by_index(begin = var_1962_begin_0, end = var_1962_end_0, end_mask = var_1962_end_mask_0, x = input_261)[name = tensor("op_1962")]; + tensor x_expanded_31 = tile(reps = x_expanded_31_reps_0, x = var_1230)[name = tensor("x_expanded_31")]; + tensor var_1244 = sub(x = var_18, y = fill_mask_31)[name = tensor("op_1244")]; + tensor var_1245 = mul(x = window_37, y = var_1244)[name = tensor("op_1245")]; + tensor var_1246 = mul(x = x_expanded_31, y = fill_mask_31)[name = tensor("op_1246")]; + tensor filled_window_31 = add(x = var_1245, y = var_1246)[name = tensor("filled_window_31")]; + tensor var_1248 = greater_equal(x = filled_31, y = var_17)[name = tensor("op_1248")]; + tensor var_1249_dtype_0 = const()[name = tensor("op_1249_dtype_0"), val = tensor("fp32")]; + tensor var_1250 = const()[name = tensor("op_1250"), val = tensor([1, 1, 1])]; + tensor var_1249 = cast(dtype = var_1249_dtype_0, x = var_1248)[name = tensor("cast_160")]; + tensor use_roll_31 = reshape(shape = var_1250, x = var_1249)[name = tensor("use_roll_31")]; + tensor var_1252 = sub(x = var_18, y = use_roll_31)[name = tensor("op_1252")]; + tensor var_1253 = mul(x = filled_window_31, y = var_1252)[name = tensor("op_1253")]; + tensor var_1254 = mul(x = rolled_31, y = use_roll_31)[name = tensor("op_1254")]; + tensor window_39 = add(x = var_1253, y = var_1254)[name = tensor("window_39")]; + tensor var_1256 = const()[name = tensor("op_1256"), val = tensor(0x1p+0)]; + tensor var_1257 = add(x = filled_31, y = var_1256)[name = tensor("op_1257")]; + tensor clip_31 = clip(alpha = var_22, beta = var_17, x = var_1257)[name = tensor("clip_31")]; + tensor var_1260_begin_0 = const()[name = tensor("op_1260_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1260_end_0 = const()[name = tensor("op_1260_end_0"), val = tensor([1, 2, 256])]; + tensor var_1260_end_mask_0 = const()[name = tensor("op_1260_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1260 = slice_by_index(begin = var_1260_begin_0, end = var_1260_end_0, end_mask = var_1260_end_mask_0, x = x_21)[name = tensor("op_1260")]; + tensor var_1263_begin_0 = const()[name = tensor("op_1263_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1263_end_0 = const()[name = tensor("op_1263_end_0"), val = tensor([1, 16, 256])]; + tensor var_1263_end_mask_0 = const()[name = tensor("op_1263_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1263 = slice_by_index(begin = var_1263_begin_0, end = var_1263_end_0, end_mask = var_1263_end_mask_0, x = window_39)[name = tensor("op_1263")]; tensor rolled_33_interleave_0 = const()[name = tensor("rolled_33_interleave_0"), val = tensor(false)]; - tensor rolled_33 = concat(axis = var_34, interleave = rolled_33_interleave_0, values = (var_1962, var_1959))[name = tensor("rolled_33")]; + tensor rolled_33 = concat(axis = var_34, interleave = rolled_33_interleave_0, values = (var_1263, var_1260))[name = tensor("rolled_33")]; tensor clip_32 = clip(alpha = var_22, beta = var_21, x = clip_31)[name = tensor("clip_32")]; - tensor var_1967 = equal(x = encoder__conv_positions, y = clip_32)[name = tensor("op_1967")]; - tensor var_1968_dtype_0 = const()[name = tensor("op_1968_dtype_0"), val = tensor("fp32")]; - tensor var_1969 = const()[name = tensor("op_1969"), val = tensor([1, 16, 1])]; - tensor var_1968 = cast(dtype = var_1968_dtype_0, x = var_1967)[name = tensor("cast_159")]; - tensor fill_mask_33 = reshape(shape = var_1969, x = var_1968)[name = tensor("fill_mask_33")]; + tensor var_1268 = equal(x = encoder__conv_positions, y = clip_32)[name = tensor("op_1268")]; + tensor var_1269_dtype_0 = const()[name = tensor("op_1269_dtype_0"), val = tensor("fp32")]; + tensor var_1270 = const()[name = tensor("op_1270"), val = tensor([1, 16, 1])]; + tensor var_1269 = cast(dtype = var_1269_dtype_0, x = var_1268)[name = tensor("cast_159")]; + tensor fill_mask_33 = reshape(shape = var_1270, x = var_1269)[name = tensor("fill_mask_33")]; tensor x_expanded_33_reps_0 = const()[name = tensor("x_expanded_33_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_33 = tile(reps = x_expanded_33_reps_0, x = var_1959)[name = tensor("x_expanded_33")]; - tensor var_1973 = sub(x = var_18, y = fill_mask_33)[name = tensor("op_1973")]; - tensor var_1974 = mul(x = input_261, y = var_1973)[name = tensor("op_1974")]; - tensor var_1975 = mul(x = x_expanded_33, y = fill_mask_33)[name = tensor("op_1975")]; - tensor filled_window_33 = add(x = var_1974, y = var_1975)[name = tensor("filled_window_33")]; - tensor var_1977 = greater_equal(x = clip_31, y = var_17)[name = tensor("op_1977")]; - tensor var_1978_dtype_0 = const()[name = tensor("op_1978_dtype_0"), val = tensor("fp32")]; - tensor var_1979 = const()[name = tensor("op_1979"), val = tensor([1, 1, 1])]; - tensor var_1978 = cast(dtype = var_1978_dtype_0, x = var_1977)[name = tensor("cast_158")]; - tensor use_roll_33 = reshape(shape = var_1979, x = var_1978)[name = tensor("use_roll_33")]; - tensor var_1981 = sub(x = var_18, y = use_roll_33)[name = tensor("op_1981")]; - tensor var_1982 = mul(x = filled_window_33, y = var_1981)[name = tensor("op_1982")]; - tensor var_1983 = mul(x = rolled_33, y = use_roll_33)[name = tensor("op_1983")]; - tensor input_271 = add(x = var_1982, y = var_1983)[name = tensor("input_271")]; - tensor var_1985 = const()[name = tensor("op_1985"), val = tensor(0x1p+0)]; - tensor var_1986 = add(x = clip_31, y = var_1985)[name = tensor("op_1986")]; - tensor clip_33 = clip(alpha = var_22, beta = var_17, x = var_1986)[name = tensor("clip_33")]; - tensor x_49_axes_0 = const()[name = tensor("x_49_axes_0"), val = tensor([-1])]; - tensor x_49 = layer_norm(axes = x_49_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_271)[name = tensor("x_49")]; - tensor input_273_perm_0 = const()[name = tensor("input_273_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_111_pad_type_0 = const()[name = tensor("inputs_111_pad_type_0"), val = tensor("valid")]; - tensor inputs_111_strides_0 = const()[name = tensor("inputs_111_strides_0"), val = tensor([1])]; - tensor inputs_111_pad_0 = const()[name = tensor("inputs_111_pad_0"), val = tensor([0, 0])]; - tensor inputs_111_dilations_0 = const()[name = tensor("inputs_111_dilations_0"), val = tensor([1])]; - tensor inputs_111_groups_0 = const()[name = tensor("inputs_111_groups_0"), val = tensor(1)]; - tensor input_273 = transpose(perm = input_273_perm_0, x = x_49)[name = tensor("transpose_38")]; - tensor inputs_111 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_111_dilations_0, groups = inputs_111_groups_0, pad = inputs_111_pad_0, pad_type = inputs_111_pad_type_0, strides = inputs_111_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_273)[name = tensor("inputs_111")]; - tensor var_2007_split_sizes_0 = const()[name = tensor("op_2007_split_sizes_0"), val = tensor([256, 256])]; - tensor var_2007_axis_0 = const()[name = tensor("op_2007_axis_0"), val = tensor(1)]; - tensor var_2007_0, tensor var_2007_1 = split(axis = var_2007_axis_0, split_sizes = var_2007_split_sizes_0, x = inputs_111)[name = tensor("op_2007")]; - tensor var_2009 = sigmoid(x = var_2007_1)[name = tensor("op_2009")]; - tensor inputs_113 = mul(x = var_2007_0, y = var_2009)[name = tensor("inputs_113")]; - tensor outputs_aug_33_pad_type_0 = const()[name = tensor("outputs_aug_33_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_33_pad_0 = const()[name = tensor("outputs_aug_33_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_33_groups_0 = const()[name = tensor("outputs_aug_33_groups_0"), val = tensor(256)]; - tensor outputs_aug_33_strides_0 = const()[name = tensor("outputs_aug_33_strides_0"), val = tensor([1])]; - tensor outputs_aug_33_dilations_0 = const()[name = tensor("outputs_aug_33_dilations_0"), val = tensor([1])]; - tensor outputs_aug_33 = conv(dilations = outputs_aug_33_dilations_0, groups = outputs_aug_33_groups_0, pad = outputs_aug_33_pad_0, pad_type = outputs_aug_33_pad_type_0, strides = outputs_aug_33_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_113)[name = tensor("outputs_aug_33")]; - tensor input_275_begin_0 = const()[name = tensor("input_275_begin_0"), val = tensor([0, 0, 0])]; - tensor input_275_end_0 = const()[name = tensor("input_275_end_0"), val = tensor([1, 256, 16])]; - tensor input_275_end_mask_0 = const()[name = tensor("input_275_end_mask_0"), val = tensor([true, true, false])]; - tensor input_275 = slice_by_index(begin = input_275_begin_0, end = input_275_end_0, end_mask = input_275_end_mask_0, x = outputs_aug_33)[name = tensor("input_275")]; - tensor inputs_115 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_275)[name = tensor("inputs_115")]; - tensor input_277 = silu(x = inputs_115)[name = tensor("input_277")]; - tensor input_279_pad_type_0 = const()[name = tensor("input_279_pad_type_0"), val = tensor("valid")]; - tensor input_279_strides_0 = const()[name = tensor("input_279_strides_0"), val = tensor([1])]; - tensor input_279_pad_0 = const()[name = tensor("input_279_pad_0"), val = tensor([0, 0])]; - tensor input_279_dilations_0 = const()[name = tensor("input_279_dilations_0"), val = tensor([1])]; - tensor input_279_groups_0 = const()[name = tensor("input_279_groups_0"), val = tensor(1)]; - tensor input_279 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_279_dilations_0, groups = input_279_groups_0, pad = input_279_pad_0, pad_type = input_279_pad_type_0, strides = input_279_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_277)[name = tensor("input_279")]; - tensor conv_out_33_perm_0 = const()[name = tensor("conv_out_33_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2040_begin_0 = const()[name = tensor("op_2040_begin_0"), val = tensor([0, -1, 0])]; - tensor var_2040_end_0 = const()[name = tensor("op_2040_end_0"), val = tensor([1, 16, 256])]; - tensor var_2040_end_mask_0 = const()[name = tensor("op_2040_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_33 = transpose(perm = conv_out_33_perm_0, x = input_279)[name = tensor("transpose_37")]; - tensor var_2040 = slice_by_index(begin = var_2040_begin_0, end = var_2040_end_0, end_mask = var_2040_end_mask_0, x = conv_out_33)[name = tensor("op_2040")]; - tensor var_2043_begin_0 = const()[name = tensor("op_2043_begin_0"), val = tensor([0, 2, 0])]; - tensor var_2043_end_0 = const()[name = tensor("op_2043_end_0"), val = tensor([1, 3, 256])]; - tensor var_2043_end_mask_0 = const()[name = tensor("op_2043_end_mask_0"), val = tensor([true, false, true])]; - tensor var_2043 = slice_by_index(begin = var_2043_begin_0, end = var_2043_end_0, end_mask = var_2043_end_mask_0, x = x_45)[name = tensor("op_2043")]; - tensor var_2046_begin_0 = const()[name = tensor("op_2046_begin_0"), val = tensor([0, 1, 0])]; - tensor var_2046_end_0 = const()[name = tensor("op_2046_end_0"), val = tensor([1, 16, 256])]; - tensor var_2046_end_mask_0 = const()[name = tensor("op_2046_end_mask_0"), val = tensor([true, true, true])]; - tensor var_2046 = slice_by_index(begin = var_2046_begin_0, end = var_2046_end_0, end_mask = var_2046_end_mask_0, x = input_271)[name = tensor("op_2046")]; + tensor x_expanded_33 = tile(reps = x_expanded_33_reps_0, x = var_1260)[name = tensor("x_expanded_33")]; + tensor var_1274 = sub(x = var_18, y = fill_mask_33)[name = tensor("op_1274")]; + tensor var_1275 = mul(x = window_39, y = var_1274)[name = tensor("op_1275")]; + tensor var_1276 = mul(x = x_expanded_33, y = fill_mask_33)[name = tensor("op_1276")]; + tensor filled_window_33 = add(x = var_1275, y = var_1276)[name = tensor("filled_window_33")]; + tensor var_1278 = greater_equal(x = clip_31, y = var_17)[name = tensor("op_1278")]; + tensor var_1279_dtype_0 = const()[name = tensor("op_1279_dtype_0"), val = tensor("fp32")]; + tensor var_1280 = const()[name = tensor("op_1280"), val = tensor([1, 1, 1])]; + tensor var_1279 = cast(dtype = var_1279_dtype_0, x = var_1278)[name = tensor("cast_158")]; + tensor use_roll_33 = reshape(shape = var_1280, x = var_1279)[name = tensor("use_roll_33")]; + tensor var_1282 = sub(x = var_18, y = use_roll_33)[name = tensor("op_1282")]; + tensor var_1283 = mul(x = filled_window_33, y = var_1282)[name = tensor("op_1283")]; + tensor var_1284 = mul(x = rolled_33, y = use_roll_33)[name = tensor("op_1284")]; + tensor window_41 = add(x = var_1283, y = var_1284)[name = tensor("window_41")]; + tensor var_1286 = const()[name = tensor("op_1286"), val = tensor(0x1p+0)]; + tensor var_1287 = add(x = clip_31, y = var_1286)[name = tensor("op_1287")]; + tensor clip_33 = clip(alpha = var_22, beta = var_17, x = var_1287)[name = tensor("clip_33")]; + tensor var_1290_begin_0 = const()[name = tensor("op_1290_begin_0"), val = tensor([0, 2, 0])]; + tensor var_1290_end_0 = const()[name = tensor("op_1290_end_0"), val = tensor([1, 3, 256])]; + tensor var_1290_end_mask_0 = const()[name = tensor("op_1290_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1290 = slice_by_index(begin = var_1290_begin_0, end = var_1290_end_0, end_mask = var_1290_end_mask_0, x = x_21)[name = tensor("op_1290")]; + tensor var_1293_begin_0 = const()[name = tensor("op_1293_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1293_end_0 = const()[name = tensor("op_1293_end_0"), val = tensor([1, 16, 256])]; + tensor var_1293_end_mask_0 = const()[name = tensor("op_1293_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1293 = slice_by_index(begin = var_1293_begin_0, end = var_1293_end_0, end_mask = var_1293_end_mask_0, x = window_41)[name = tensor("op_1293")]; tensor rolled_35_interleave_0 = const()[name = tensor("rolled_35_interleave_0"), val = tensor(false)]; - tensor rolled_35 = concat(axis = var_34, interleave = rolled_35_interleave_0, values = (var_2046, var_2043))[name = tensor("rolled_35")]; + tensor rolled_35 = concat(axis = var_34, interleave = rolled_35_interleave_0, values = (var_1293, var_1290))[name = tensor("rolled_35")]; tensor clip_34 = clip(alpha = var_22, beta = var_21, x = clip_33)[name = tensor("clip_34")]; - tensor var_2051 = equal(x = encoder__conv_positions, y = clip_34)[name = tensor("op_2051")]; - tensor var_2052_dtype_0 = const()[name = tensor("op_2052_dtype_0"), val = tensor("fp32")]; - tensor var_2053 = const()[name = tensor("op_2053"), val = tensor([1, 16, 1])]; - tensor var_2052 = cast(dtype = var_2052_dtype_0, x = var_2051)[name = tensor("cast_157")]; - tensor fill_mask_35 = reshape(shape = var_2053, x = var_2052)[name = tensor("fill_mask_35")]; + tensor var_1298 = equal(x = encoder__conv_positions, y = clip_34)[name = tensor("op_1298")]; + tensor var_1299_dtype_0 = const()[name = tensor("op_1299_dtype_0"), val = tensor("fp32")]; + tensor var_1300 = const()[name = tensor("op_1300"), val = tensor([1, 16, 1])]; + tensor var_1299 = cast(dtype = var_1299_dtype_0, x = var_1298)[name = tensor("cast_157")]; + tensor fill_mask_35 = reshape(shape = var_1300, x = var_1299)[name = tensor("fill_mask_35")]; tensor x_expanded_35_reps_0 = const()[name = tensor("x_expanded_35_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_35 = tile(reps = x_expanded_35_reps_0, x = var_2043)[name = tensor("x_expanded_35")]; - tensor var_2057 = sub(x = var_18, y = fill_mask_35)[name = tensor("op_2057")]; - tensor var_2058 = mul(x = input_271, y = var_2057)[name = tensor("op_2058")]; - tensor var_2059 = mul(x = x_expanded_35, y = fill_mask_35)[name = tensor("op_2059")]; - tensor filled_window_35 = add(x = var_2058, y = var_2059)[name = tensor("filled_window_35")]; - tensor var_2061 = greater_equal(x = clip_33, y = var_17)[name = tensor("op_2061")]; - tensor var_2062_dtype_0 = const()[name = tensor("op_2062_dtype_0"), val = tensor("fp32")]; - tensor var_2063 = const()[name = tensor("op_2063"), val = tensor([1, 1, 1])]; - tensor var_2062 = cast(dtype = var_2062_dtype_0, x = var_2061)[name = tensor("cast_156")]; - tensor use_roll_35 = reshape(shape = var_2063, x = var_2062)[name = tensor("use_roll_35")]; - tensor var_2065 = sub(x = var_18, y = use_roll_35)[name = tensor("op_2065")]; - tensor var_2066 = mul(x = filled_window_35, y = var_2065)[name = tensor("op_2066")]; - tensor var_2067 = mul(x = rolled_35, y = use_roll_35)[name = tensor("op_2067")]; - tensor input_281 = add(x = var_2066, y = var_2067)[name = tensor("input_281")]; - tensor var_2069 = const()[name = tensor("op_2069"), val = tensor(0x1p+0)]; - tensor var_2070 = add(x = clip_33, y = var_2069)[name = tensor("op_2070")]; - tensor clip_35 = clip(alpha = var_22, beta = var_17, x = var_2070)[name = tensor("clip_35")]; - tensor x_51_axes_0 = const()[name = tensor("x_51_axes_0"), val = tensor([-1])]; - tensor x_51 = layer_norm(axes = x_51_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_281)[name = tensor("x_51")]; - tensor input_283_perm_0 = const()[name = tensor("input_283_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_117_pad_type_0 = const()[name = tensor("inputs_117_pad_type_0"), val = tensor("valid")]; - tensor inputs_117_strides_0 = const()[name = tensor("inputs_117_strides_0"), val = tensor([1])]; - tensor inputs_117_pad_0 = const()[name = tensor("inputs_117_pad_0"), val = tensor([0, 0])]; - tensor inputs_117_dilations_0 = const()[name = tensor("inputs_117_dilations_0"), val = tensor([1])]; - tensor inputs_117_groups_0 = const()[name = tensor("inputs_117_groups_0"), val = tensor(1)]; - tensor input_283 = transpose(perm = input_283_perm_0, x = x_51)[name = tensor("transpose_36")]; - tensor inputs_117 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_117_dilations_0, groups = inputs_117_groups_0, pad = inputs_117_pad_0, pad_type = inputs_117_pad_type_0, strides = inputs_117_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_283)[name = tensor("inputs_117")]; - tensor var_2091_split_sizes_0 = const()[name = tensor("op_2091_split_sizes_0"), val = tensor([256, 256])]; - tensor var_2091_axis_0 = const()[name = tensor("op_2091_axis_0"), val = tensor(1)]; - tensor var_2091_0, tensor var_2091_1 = split(axis = var_2091_axis_0, split_sizes = var_2091_split_sizes_0, x = inputs_117)[name = tensor("op_2091")]; - tensor var_2093 = sigmoid(x = var_2091_1)[name = tensor("op_2093")]; - tensor inputs_119 = mul(x = var_2091_0, y = var_2093)[name = tensor("inputs_119")]; - tensor outputs_aug_35_pad_type_0 = const()[name = tensor("outputs_aug_35_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_35_pad_0 = const()[name = tensor("outputs_aug_35_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_35_groups_0 = const()[name = tensor("outputs_aug_35_groups_0"), val = tensor(256)]; - tensor outputs_aug_35_strides_0 = const()[name = tensor("outputs_aug_35_strides_0"), val = tensor([1])]; - tensor outputs_aug_35_dilations_0 = const()[name = tensor("outputs_aug_35_dilations_0"), val = tensor([1])]; - tensor outputs_aug_35 = conv(dilations = outputs_aug_35_dilations_0, groups = outputs_aug_35_groups_0, pad = outputs_aug_35_pad_0, pad_type = outputs_aug_35_pad_type_0, strides = outputs_aug_35_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_119)[name = tensor("outputs_aug_35")]; - tensor input_285_begin_0 = const()[name = tensor("input_285_begin_0"), val = tensor([0, 0, 0])]; - tensor input_285_end_0 = const()[name = tensor("input_285_end_0"), val = tensor([1, 256, 16])]; - tensor input_285_end_mask_0 = const()[name = tensor("input_285_end_mask_0"), val = tensor([true, true, false])]; - tensor input_285 = slice_by_index(begin = input_285_begin_0, end = input_285_end_0, end_mask = input_285_end_mask_0, x = outputs_aug_35)[name = tensor("input_285")]; - tensor inputs_121 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_285)[name = tensor("inputs_121")]; - tensor input_287 = silu(x = inputs_121)[name = tensor("input_287")]; - tensor input_289_pad_type_0 = const()[name = tensor("input_289_pad_type_0"), val = tensor("valid")]; - tensor input_289_strides_0 = const()[name = tensor("input_289_strides_0"), val = tensor([1])]; - tensor input_289_pad_0 = const()[name = tensor("input_289_pad_0"), val = tensor([0, 0])]; - tensor input_289_dilations_0 = const()[name = tensor("input_289_dilations_0"), val = tensor([1])]; - tensor input_289_groups_0 = const()[name = tensor("input_289_groups_0"), val = tensor(1)]; - tensor input_289 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_289_dilations_0, groups = input_289_groups_0, pad = input_289_pad_0, pad_type = input_289_pad_type_0, strides = input_289_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_287)[name = tensor("input_289")]; - tensor conv_out_35_perm_0 = const()[name = tensor("conv_out_35_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2124_begin_0 = const()[name = tensor("op_2124_begin_0"), val = tensor([0, -1, 0])]; - tensor var_2124_end_0 = const()[name = tensor("op_2124_end_0"), val = tensor([1, 16, 256])]; - tensor var_2124_end_mask_0 = const()[name = tensor("op_2124_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_35 = transpose(perm = conv_out_35_perm_0, x = input_289)[name = tensor("transpose_35")]; - tensor var_2124 = slice_by_index(begin = var_2124_begin_0, end = var_2124_end_0, end_mask = var_2124_end_mask_0, x = conv_out_35)[name = tensor("op_2124")]; - tensor var_2127_begin_0 = const()[name = tensor("op_2127_begin_0"), val = tensor([0, 3, 0])]; - tensor var_2127_end_0 = const()[name = tensor("op_2127_end_0"), val = tensor([1, 4, 256])]; - tensor var_2127_end_mask_0 = const()[name = tensor("op_2127_end_mask_0"), val = tensor([true, false, true])]; - tensor var_2127 = slice_by_index(begin = var_2127_begin_0, end = var_2127_end_0, end_mask = var_2127_end_mask_0, x = x_45)[name = tensor("op_2127")]; - tensor var_2130_begin_0 = const()[name = tensor("op_2130_begin_0"), val = tensor([0, 1, 0])]; - tensor var_2130_end_0 = const()[name = tensor("op_2130_end_0"), val = tensor([1, 16, 256])]; - tensor var_2130_end_mask_0 = const()[name = tensor("op_2130_end_mask_0"), val = tensor([true, true, true])]; - tensor var_2130 = slice_by_index(begin = var_2130_begin_0, end = var_2130_end_0, end_mask = var_2130_end_mask_0, x = input_281)[name = tensor("op_2130")]; + tensor x_expanded_35 = tile(reps = x_expanded_35_reps_0, x = var_1290)[name = tensor("x_expanded_35")]; + tensor var_1304 = sub(x = var_18, y = fill_mask_35)[name = tensor("op_1304")]; + tensor var_1305 = mul(x = window_41, y = var_1304)[name = tensor("op_1305")]; + tensor var_1306 = mul(x = x_expanded_35, y = fill_mask_35)[name = tensor("op_1306")]; + tensor filled_window_35 = add(x = var_1305, y = var_1306)[name = tensor("filled_window_35")]; + tensor var_1308 = greater_equal(x = clip_33, y = var_17)[name = tensor("op_1308")]; + tensor var_1309_dtype_0 = const()[name = tensor("op_1309_dtype_0"), val = tensor("fp32")]; + tensor var_1310 = const()[name = tensor("op_1310"), val = tensor([1, 1, 1])]; + tensor var_1309 = cast(dtype = var_1309_dtype_0, x = var_1308)[name = tensor("cast_156")]; + tensor use_roll_35 = reshape(shape = var_1310, x = var_1309)[name = tensor("use_roll_35")]; + tensor var_1312 = sub(x = var_18, y = use_roll_35)[name = tensor("op_1312")]; + tensor var_1313 = mul(x = filled_window_35, y = var_1312)[name = tensor("op_1313")]; + tensor var_1314 = mul(x = rolled_35, y = use_roll_35)[name = tensor("op_1314")]; + tensor window_43 = add(x = var_1313, y = var_1314)[name = tensor("window_43")]; + tensor var_1316 = const()[name = tensor("op_1316"), val = tensor(0x1p+0)]; + tensor var_1317 = add(x = clip_33, y = var_1316)[name = tensor("op_1317")]; + tensor clip_35 = clip(alpha = var_22, beta = var_17, x = var_1317)[name = tensor("clip_35")]; + tensor var_1320_begin_0 = const()[name = tensor("op_1320_begin_0"), val = tensor([0, 3, 0])]; + tensor var_1320_end_0 = const()[name = tensor("op_1320_end_0"), val = tensor([1, 4, 256])]; + tensor var_1320_end_mask_0 = const()[name = tensor("op_1320_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1320 = slice_by_index(begin = var_1320_begin_0, end = var_1320_end_0, end_mask = var_1320_end_mask_0, x = x_21)[name = tensor("op_1320")]; + tensor var_1323_begin_0 = const()[name = tensor("op_1323_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1323_end_0 = const()[name = tensor("op_1323_end_0"), val = tensor([1, 16, 256])]; + tensor var_1323_end_mask_0 = const()[name = tensor("op_1323_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1323 = slice_by_index(begin = var_1323_begin_0, end = var_1323_end_0, end_mask = var_1323_end_mask_0, x = window_43)[name = tensor("op_1323")]; tensor rolled_37_interleave_0 = const()[name = tensor("rolled_37_interleave_0"), val = tensor(false)]; - tensor rolled_37 = concat(axis = var_34, interleave = rolled_37_interleave_0, values = (var_2130, var_2127))[name = tensor("rolled_37")]; + tensor rolled_37 = concat(axis = var_34, interleave = rolled_37_interleave_0, values = (var_1323, var_1320))[name = tensor("rolled_37")]; tensor clip_36 = clip(alpha = var_22, beta = var_21, x = clip_35)[name = tensor("clip_36")]; - tensor var_2135 = equal(x = encoder__conv_positions, y = clip_36)[name = tensor("op_2135")]; - tensor var_2136_dtype_0 = const()[name = tensor("op_2136_dtype_0"), val = tensor("fp32")]; - tensor var_2137 = const()[name = tensor("op_2137"), val = tensor([1, 16, 1])]; - tensor var_2136 = cast(dtype = var_2136_dtype_0, x = var_2135)[name = tensor("cast_155")]; - tensor fill_mask_37 = reshape(shape = var_2137, x = var_2136)[name = tensor("fill_mask_37")]; + tensor var_1328 = equal(x = encoder__conv_positions, y = clip_36)[name = tensor("op_1328")]; + tensor var_1329_dtype_0 = const()[name = tensor("op_1329_dtype_0"), val = tensor("fp32")]; + tensor var_1330 = const()[name = tensor("op_1330"), val = tensor([1, 16, 1])]; + tensor var_1329 = cast(dtype = var_1329_dtype_0, x = var_1328)[name = tensor("cast_155")]; + tensor fill_mask_37 = reshape(shape = var_1330, x = var_1329)[name = tensor("fill_mask_37")]; tensor x_expanded_37_reps_0 = const()[name = tensor("x_expanded_37_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_37 = tile(reps = x_expanded_37_reps_0, x = var_2127)[name = tensor("x_expanded_37")]; - tensor var_2141 = sub(x = var_18, y = fill_mask_37)[name = tensor("op_2141")]; - tensor var_2142 = mul(x = input_281, y = var_2141)[name = tensor("op_2142")]; - tensor var_2143 = mul(x = x_expanded_37, y = fill_mask_37)[name = tensor("op_2143")]; - tensor filled_window_37 = add(x = var_2142, y = var_2143)[name = tensor("filled_window_37")]; - tensor var_2145 = greater_equal(x = clip_35, y = var_17)[name = tensor("op_2145")]; - tensor var_2146_dtype_0 = const()[name = tensor("op_2146_dtype_0"), val = tensor("fp32")]; - tensor var_2147 = const()[name = tensor("op_2147"), val = tensor([1, 1, 1])]; - tensor var_2146 = cast(dtype = var_2146_dtype_0, x = var_2145)[name = tensor("cast_154")]; - tensor use_roll_37 = reshape(shape = var_2147, x = var_2146)[name = tensor("use_roll_37")]; - tensor var_2149 = sub(x = var_18, y = use_roll_37)[name = tensor("op_2149")]; - tensor var_2150 = mul(x = filled_window_37, y = var_2149)[name = tensor("op_2150")]; - tensor var_2151 = mul(x = rolled_37, y = use_roll_37)[name = tensor("op_2151")]; - tensor input_291 = add(x = var_2150, y = var_2151)[name = tensor("input_291")]; - tensor var_2153 = const()[name = tensor("op_2153"), val = tensor(0x1p+0)]; - tensor var_2154 = add(x = clip_35, y = var_2153)[name = tensor("op_2154")]; - tensor clip_37 = clip(alpha = var_22, beta = var_17, x = var_2154)[name = tensor("clip_37")]; - tensor x_53_axes_0 = const()[name = tensor("x_53_axes_0"), val = tensor([-1])]; - tensor x_53 = layer_norm(axes = x_53_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_291)[name = tensor("x_53")]; - tensor input_293_perm_0 = const()[name = tensor("input_293_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_123_pad_type_0 = const()[name = tensor("inputs_123_pad_type_0"), val = tensor("valid")]; - tensor inputs_123_strides_0 = const()[name = tensor("inputs_123_strides_0"), val = tensor([1])]; - tensor inputs_123_pad_0 = const()[name = tensor("inputs_123_pad_0"), val = tensor([0, 0])]; - tensor inputs_123_dilations_0 = const()[name = tensor("inputs_123_dilations_0"), val = tensor([1])]; - tensor inputs_123_groups_0 = const()[name = tensor("inputs_123_groups_0"), val = tensor(1)]; - tensor input_293 = transpose(perm = input_293_perm_0, x = x_53)[name = tensor("transpose_34")]; - tensor inputs_123 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_123_dilations_0, groups = inputs_123_groups_0, pad = inputs_123_pad_0, pad_type = inputs_123_pad_type_0, strides = inputs_123_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_293)[name = tensor("inputs_123")]; - tensor var_2175_split_sizes_0 = const()[name = tensor("op_2175_split_sizes_0"), val = tensor([256, 256])]; - tensor var_2175_axis_0 = const()[name = tensor("op_2175_axis_0"), val = tensor(1)]; - tensor var_2175_0, tensor var_2175_1 = split(axis = var_2175_axis_0, split_sizes = var_2175_split_sizes_0, x = inputs_123)[name = tensor("op_2175")]; - tensor var_2177 = sigmoid(x = var_2175_1)[name = tensor("op_2177")]; - tensor inputs_125 = mul(x = var_2175_0, y = var_2177)[name = tensor("inputs_125")]; - tensor outputs_aug_37_pad_type_0 = const()[name = tensor("outputs_aug_37_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_37_pad_0 = const()[name = tensor("outputs_aug_37_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_37_groups_0 = const()[name = tensor("outputs_aug_37_groups_0"), val = tensor(256)]; - tensor outputs_aug_37_strides_0 = const()[name = tensor("outputs_aug_37_strides_0"), val = tensor([1])]; - tensor outputs_aug_37_dilations_0 = const()[name = tensor("outputs_aug_37_dilations_0"), val = tensor([1])]; - tensor outputs_aug_37 = conv(dilations = outputs_aug_37_dilations_0, groups = outputs_aug_37_groups_0, pad = outputs_aug_37_pad_0, pad_type = outputs_aug_37_pad_type_0, strides = outputs_aug_37_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_125)[name = tensor("outputs_aug_37")]; - tensor input_295_begin_0 = const()[name = tensor("input_295_begin_0"), val = tensor([0, 0, 0])]; - tensor input_295_end_0 = const()[name = tensor("input_295_end_0"), val = tensor([1, 256, 16])]; - tensor input_295_end_mask_0 = const()[name = tensor("input_295_end_mask_0"), val = tensor([true, true, false])]; - tensor input_295 = slice_by_index(begin = input_295_begin_0, end = input_295_end_0, end_mask = input_295_end_mask_0, x = outputs_aug_37)[name = tensor("input_295")]; - tensor inputs_127 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_295)[name = tensor("inputs_127")]; - tensor input_297 = silu(x = inputs_127)[name = tensor("input_297")]; - tensor input_299_pad_type_0 = const()[name = tensor("input_299_pad_type_0"), val = tensor("valid")]; - tensor input_299_strides_0 = const()[name = tensor("input_299_strides_0"), val = tensor([1])]; - tensor input_299_pad_0 = const()[name = tensor("input_299_pad_0"), val = tensor([0, 0])]; - tensor input_299_dilations_0 = const()[name = tensor("input_299_dilations_0"), val = tensor([1])]; - tensor input_299_groups_0 = const()[name = tensor("input_299_groups_0"), val = tensor(1)]; - tensor input_299 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_299_dilations_0, groups = input_299_groups_0, pad = input_299_pad_0, pad_type = input_299_pad_type_0, strides = input_299_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_297)[name = tensor("input_299")]; - tensor conv_out_37_perm_0 = const()[name = tensor("conv_out_37_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2208_begin_0 = const()[name = tensor("op_2208_begin_0"), val = tensor([0, -1, 0])]; - tensor var_2208_end_0 = const()[name = tensor("op_2208_end_0"), val = tensor([1, 16, 256])]; - tensor var_2208_end_mask_0 = const()[name = tensor("op_2208_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_37 = transpose(perm = conv_out_37_perm_0, x = input_299)[name = tensor("transpose_33")]; - tensor var_2208 = slice_by_index(begin = var_2208_begin_0, end = var_2208_end_0, end_mask = var_2208_end_mask_0, x = conv_out_37)[name = tensor("op_2208")]; - tensor var_2211_begin_0 = const()[name = tensor("op_2211_begin_0"), val = tensor([0, 4, 0])]; - tensor var_2211_end_0 = const()[name = tensor("op_2211_end_0"), val = tensor([1, 1, 256])]; - tensor var_2211_end_mask_0 = const()[name = tensor("op_2211_end_mask_0"), val = tensor([true, true, true])]; - tensor var_2211 = slice_by_index(begin = var_2211_begin_0, end = var_2211_end_0, end_mask = var_2211_end_mask_0, x = x_45)[name = tensor("op_2211")]; - tensor var_2214_begin_0 = const()[name = tensor("op_2214_begin_0"), val = tensor([0, 1, 0])]; - tensor var_2214_end_0 = const()[name = tensor("op_2214_end_0"), val = tensor([1, 16, 256])]; - tensor var_2214_end_mask_0 = const()[name = tensor("op_2214_end_mask_0"), val = tensor([true, true, true])]; - tensor var_2214 = slice_by_index(begin = var_2214_begin_0, end = var_2214_end_0, end_mask = var_2214_end_mask_0, x = input_291)[name = tensor("op_2214")]; + tensor x_expanded_37 = tile(reps = x_expanded_37_reps_0, x = var_1320)[name = tensor("x_expanded_37")]; + tensor var_1334 = sub(x = var_18, y = fill_mask_37)[name = tensor("op_1334")]; + tensor var_1335 = mul(x = window_43, y = var_1334)[name = tensor("op_1335")]; + tensor var_1336 = mul(x = x_expanded_37, y = fill_mask_37)[name = tensor("op_1336")]; + tensor filled_window_37 = add(x = var_1335, y = var_1336)[name = tensor("filled_window_37")]; + tensor var_1338 = greater_equal(x = clip_35, y = var_17)[name = tensor("op_1338")]; + tensor var_1339_dtype_0 = const()[name = tensor("op_1339_dtype_0"), val = tensor("fp32")]; + tensor var_1340 = const()[name = tensor("op_1340"), val = tensor([1, 1, 1])]; + tensor var_1339 = cast(dtype = var_1339_dtype_0, x = var_1338)[name = tensor("cast_154")]; + tensor use_roll_37 = reshape(shape = var_1340, x = var_1339)[name = tensor("use_roll_37")]; + tensor var_1342 = sub(x = var_18, y = use_roll_37)[name = tensor("op_1342")]; + tensor var_1343 = mul(x = filled_window_37, y = var_1342)[name = tensor("op_1343")]; + tensor var_1344 = mul(x = rolled_37, y = use_roll_37)[name = tensor("op_1344")]; + tensor window_45 = add(x = var_1343, y = var_1344)[name = tensor("window_45")]; + tensor var_1346 = const()[name = tensor("op_1346"), val = tensor(0x1p+0)]; + tensor var_1347 = add(x = clip_35, y = var_1346)[name = tensor("op_1347")]; + tensor clip_37 = clip(alpha = var_22, beta = var_17, x = var_1347)[name = tensor("clip_37")]; + tensor var_1350_begin_0 = const()[name = tensor("op_1350_begin_0"), val = tensor([0, 4, 0])]; + tensor var_1350_end_0 = const()[name = tensor("op_1350_end_0"), val = tensor([1, 1, 256])]; + tensor var_1350_end_mask_0 = const()[name = tensor("op_1350_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1350 = slice_by_index(begin = var_1350_begin_0, end = var_1350_end_0, end_mask = var_1350_end_mask_0, x = x_21)[name = tensor("op_1350")]; + tensor var_1353_begin_0 = const()[name = tensor("op_1353_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1353_end_0 = const()[name = tensor("op_1353_end_0"), val = tensor([1, 16, 256])]; + tensor var_1353_end_mask_0 = const()[name = tensor("op_1353_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1353 = slice_by_index(begin = var_1353_begin_0, end = var_1353_end_0, end_mask = var_1353_end_mask_0, x = window_45)[name = tensor("op_1353")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_34, interleave = rolled_interleave_0, values = (var_2214, var_2211))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_34, interleave = rolled_interleave_0, values = (var_1353, var_1350))[name = tensor("rolled")]; tensor clip_38 = clip(alpha = var_22, beta = var_21, x = clip_37)[name = tensor("clip_38")]; - tensor var_2219 = equal(x = encoder__conv_positions, y = clip_38)[name = tensor("op_2219")]; - tensor var_2220_dtype_0 = const()[name = tensor("op_2220_dtype_0"), val = tensor("fp32")]; - tensor var_2221 = const()[name = tensor("op_2221"), val = tensor([1, 16, 1])]; - tensor var_2220 = cast(dtype = var_2220_dtype_0, x = var_2219)[name = tensor("cast_153")]; - tensor fill_mask = reshape(shape = var_2221, x = var_2220)[name = tensor("fill_mask")]; + tensor var_1358 = equal(x = encoder__conv_positions, y = clip_38)[name = tensor("op_1358")]; + tensor var_1359_dtype_0 = const()[name = tensor("op_1359_dtype_0"), val = tensor("fp32")]; + tensor var_1360 = const()[name = tensor("op_1360"), val = tensor([1, 16, 1])]; + tensor var_1359 = cast(dtype = var_1359_dtype_0, x = var_1358)[name = tensor("cast_153")]; + tensor fill_mask = reshape(shape = var_1360, x = var_1359)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_2211)[name = tensor("x_expanded")]; - tensor var_2225 = sub(x = var_18, y = fill_mask)[name = tensor("op_2225")]; - tensor var_2226 = mul(x = input_291, y = var_2225)[name = tensor("op_2226")]; - tensor var_2227 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_2227")]; - tensor filled_window = add(x = var_2226, y = var_2227)[name = tensor("filled_window")]; - tensor var_2229 = greater_equal(x = clip_37, y = var_17)[name = tensor("op_2229")]; - tensor var_2230_dtype_0 = const()[name = tensor("op_2230_dtype_0"), val = tensor("fp32")]; - tensor var_2231 = const()[name = tensor("op_2231"), val = tensor([1, 1, 1])]; - tensor var_2230 = cast(dtype = var_2230_dtype_0, x = var_2229)[name = tensor("cast_152")]; - tensor use_roll = reshape(shape = var_2231, x = var_2230)[name = tensor("use_roll")]; - tensor var_2233 = sub(x = var_18, y = use_roll)[name = tensor("op_2233")]; - tensor var_2234 = mul(x = filled_window, y = var_2233)[name = tensor("op_2234")]; - tensor var_2235 = mul(x = rolled, y = use_roll)[name = tensor("op_2235")]; - tensor input_301 = add(x = var_2234, y = var_2235)[name = tensor("input_301")]; - tensor var_2237 = const()[name = tensor("op_2237"), val = tensor(0x1p+0)]; - tensor var_2238 = add(x = clip_37, y = var_2237)[name = tensor("op_2238")]; - tensor clip_39 = clip(alpha = var_22, beta = var_17, x = var_2238)[name = tensor("clip_39")]; - tensor x_55_axes_0 = const()[name = tensor("x_55_axes_0"), val = tensor([-1])]; - tensor x_55 = layer_norm(axes = x_55_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_301)[name = tensor("x_55")]; - tensor input_303_perm_0 = const()[name = tensor("input_303_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_129_pad_type_0 = const()[name = tensor("inputs_129_pad_type_0"), val = tensor("valid")]; - tensor inputs_129_strides_0 = const()[name = tensor("inputs_129_strides_0"), val = tensor([1])]; - tensor inputs_129_pad_0 = const()[name = tensor("inputs_129_pad_0"), val = tensor([0, 0])]; - tensor inputs_129_dilations_0 = const()[name = tensor("inputs_129_dilations_0"), val = tensor([1])]; - tensor inputs_129_groups_0 = const()[name = tensor("inputs_129_groups_0"), val = tensor(1)]; - tensor input_303 = transpose(perm = input_303_perm_0, x = x_55)[name = tensor("transpose_32")]; - tensor inputs_129 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_129_dilations_0, groups = inputs_129_groups_0, pad = inputs_129_pad_0, pad_type = inputs_129_pad_type_0, strides = inputs_129_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_303)[name = tensor("inputs_129")]; - tensor var_2259_split_sizes_0 = const()[name = tensor("op_2259_split_sizes_0"), val = tensor([256, 256])]; - tensor var_2259_axis_0 = const()[name = tensor("op_2259_axis_0"), val = tensor(1)]; - tensor var_2259_0, tensor var_2259_1 = split(axis = var_2259_axis_0, split_sizes = var_2259_split_sizes_0, x = inputs_129)[name = tensor("op_2259")]; - tensor var_2261 = sigmoid(x = var_2259_1)[name = tensor("op_2261")]; - tensor inputs_131 = mul(x = var_2259_0, y = var_2261)[name = tensor("inputs_131")]; + tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1350)[name = tensor("x_expanded")]; + tensor var_1364 = sub(x = var_18, y = fill_mask)[name = tensor("op_1364")]; + tensor var_1365 = mul(x = window_45, y = var_1364)[name = tensor("op_1365")]; + tensor var_1366 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1366")]; + tensor filled_window = add(x = var_1365, y = var_1366)[name = tensor("filled_window")]; + tensor var_1368 = greater_equal(x = clip_37, y = var_17)[name = tensor("op_1368")]; + tensor var_1369_dtype_0 = const()[name = tensor("op_1369_dtype_0"), val = tensor("fp32")]; + tensor var_1370 = const()[name = tensor("op_1370"), val = tensor([1, 1, 1])]; + tensor var_1369 = cast(dtype = var_1369_dtype_0, x = var_1368)[name = tensor("cast_152")]; + tensor use_roll = reshape(shape = var_1370, x = var_1369)[name = tensor("use_roll")]; + tensor var_1372 = sub(x = var_18, y = use_roll)[name = tensor("op_1372")]; + tensor var_1373 = mul(x = filled_window, y = var_1372)[name = tensor("op_1373")]; + tensor var_1374 = mul(x = rolled, y = use_roll)[name = tensor("op_1374")]; + tensor window = add(x = var_1373, y = var_1374)[name = tensor("window")]; + tensor var_1376 = const()[name = tensor("op_1376"), val = tensor(0x1p+0)]; + tensor var_1377 = add(x = clip_37, y = var_1376)[name = tensor("op_1377")]; + tensor clip_39 = clip(alpha = var_22, beta = var_17, x = var_1377)[name = tensor("clip_39")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_31, interleave = input_141_interleave_0, values = (window_39, window_41, window_43, window_45, window))[name = tensor("input_141")]; + tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; + tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; + tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_33_pad_type_0 = const()[name = tensor("inputs_33_pad_type_0"), val = tensor("valid")]; + tensor inputs_33_strides_0 = const()[name = tensor("inputs_33_strides_0"), val = tensor([1])]; + tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; + tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; + tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; + tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; + tensor var_1400_split_sizes_0 = const()[name = tensor("op_1400_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1400_axis_0 = const()[name = tensor("op_1400_axis_0"), val = tensor(1)]; + tensor var_1400_0, tensor var_1400_1 = split(axis = var_1400_axis_0, split_sizes = var_1400_split_sizes_0, x = inputs_33)[name = tensor("op_1400")]; + tensor var_1402 = sigmoid(x = var_1400_1)[name = tensor("op_1402")]; + tensor inputs_35 = mul(x = var_1400_0, y = var_1402)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; tensor outputs_aug_strides_0 = const()[name = tensor("outputs_aug_strides_0"), val = tensor([1])]; tensor outputs_aug_dilations_0 = const()[name = tensor("outputs_aug_dilations_0"), val = tensor([1])]; - tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_131)[name = tensor("outputs_aug")]; - tensor input_305_begin_0 = const()[name = tensor("input_305_begin_0"), val = tensor([0, 0, 0])]; - tensor input_305_end_0 = const()[name = tensor("input_305_end_0"), val = tensor([1, 256, 16])]; - tensor input_305_end_mask_0 = const()[name = tensor("input_305_end_mask_0"), val = tensor([true, true, false])]; - tensor input_305 = slice_by_index(begin = input_305_begin_0, end = input_305_end_0, end_mask = input_305_end_mask_0, x = outputs_aug)[name = tensor("input_305")]; - tensor inputs_133 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_305)[name = tensor("inputs_133")]; - tensor input_307 = silu(x = inputs_133)[name = tensor("input_307")]; - tensor input_309_pad_type_0 = const()[name = tensor("input_309_pad_type_0"), val = tensor("valid")]; - tensor input_309_strides_0 = const()[name = tensor("input_309_strides_0"), val = tensor([1])]; - tensor input_309_pad_0 = const()[name = tensor("input_309_pad_0"), val = tensor([0, 0])]; - tensor input_309_dilations_0 = const()[name = tensor("input_309_dilations_0"), val = tensor([1])]; - tensor input_309_groups_0 = const()[name = tensor("input_309_groups_0"), val = tensor(1)]; - tensor input_309 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_309_dilations_0, groups = input_309_groups_0, pad = input_309_pad_0, pad_type = input_309_pad_type_0, strides = input_309_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_307)[name = tensor("input_309")]; - tensor conv_out_39_perm_0 = const()[name = tensor("conv_out_39_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2292_begin_0 = const()[name = tensor("op_2292_begin_0"), val = tensor([0, -1, 0])]; - tensor var_2292_end_0 = const()[name = tensor("op_2292_end_0"), val = tensor([1, 16, 256])]; - tensor var_2292_end_mask_0 = const()[name = tensor("op_2292_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_39 = transpose(perm = conv_out_39_perm_0, x = input_309)[name = tensor("transpose_31")]; - tensor var_2292 = slice_by_index(begin = var_2292_begin_0, end = var_2292_end_0, end_mask = var_2292_end_mask_0, x = conv_out_39)[name = tensor("op_2292")]; - tensor var_2295_interleave_0 = const()[name = tensor("op_2295_interleave_0"), val = tensor(false)]; - tensor var_2295 = concat(axis = var_34, interleave = var_2295_interleave_0, values = (var_1956, var_2040, var_2124, var_2208, var_2292))[name = tensor("op_2295")]; - tensor input_311 = add(x = x_45, y = var_2295)[name = tensor("input_311")]; - tensor input_313_axes_0 = const()[name = tensor("input_313_axes_0"), val = tensor([-1])]; - tensor input_313 = layer_norm(axes = input_313_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_311)[name = tensor("input_313")]; - tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_313)[name = tensor("linear_35")]; - tensor input_315 = silu(x = inputs)[name = tensor("input_315")]; - tensor input_319 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_315)[name = tensor("linear_36")]; - tensor var_2318 = const()[name = tensor("op_2318"), val = tensor(0x1p-1)]; - tensor var_2319 = mul(x = input_319, y = var_2318)[name = tensor("op_2319")]; - tensor input_321 = add(x = var_2319, y = input_311)[name = tensor("input_321")]; - tensor x_57_axes_0 = const()[name = tensor("x_57_axes_0"), val = tensor([-1])]; - tensor x_57 = layer_norm(axes = x_57_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_36, gamma = encoder_layer_norm_3_weight, x = input_321)[name = tensor("x_57")]; + tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_35)[name = tensor("outputs_aug")]; + tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; + tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([5, 256, 16])]; + tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; + tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug)[name = tensor("input_145")]; + tensor inputs_37 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_145)[name = tensor("inputs_37")]; + tensor input_147 = silu(x = inputs_37)[name = tensor("input_147")]; + tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; + tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; + tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; + tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; + tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; + tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; + tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1433_begin_0 = const()[name = tensor("op_1433_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1433_end_0 = const()[name = tensor("op_1433_end_0"), val = tensor([5, 16, 256])]; + tensor var_1433_end_mask_0 = const()[name = tensor("op_1433_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_1433 = slice_by_index(begin = var_1433_begin_0, end = var_1433_end_0, end_mask = var_1433_end_mask_0, x = conv_out_7)[name = tensor("op_1433")]; + tensor var_1435_perm_0 = const()[name = tensor("op_1435_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1435 = transpose(perm = var_1435_perm_0, x = var_1433)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_1435)[name = tensor("input_151")]; + tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; + tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; + tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; + tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; + tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; + tensor var_1458 = const()[name = tensor("op_1458"), val = tensor(0x1p-1)]; + tensor var_1459 = mul(x = input_159, y = var_1458)[name = tensor("op_1459")]; + tensor input_161 = add(x = var_1459, y = input_151)[name = tensor("input_161")]; + tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; + tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_36, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; tensor cat_interleave_0 = const()[name = tensor("cat_interleave_0"), val = tensor(false)]; - tensor x_ct = transpose(perm = x_ct_perm_0, x = x_57)[name = tensor("transpose_30")]; + tensor x_ct = transpose(perm = x_ct_perm_0, x = x_25)[name = tensor("transpose_30")]; tensor cat = concat(axis = var_28, interleave = cat_interleave_0, values = (cnn_window, x_ct))[name = tensor("cat")]; tensor conv_out_pad_type_0 = const()[name = tensor("conv_out_pad_type_0"), val = tensor("valid")]; tensor conv_out_strides_0 = const()[name = tensor("conv_out_strides_0"), val = tensor([1])]; @@ -2097,44 +1482,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_2337_begin_0 = const()[name = tensor("op_2337_begin_0"), val = tensor([0, 0, 5])]; - tensor var_2337_end_0 = const()[name = tensor("op_2337_end_0"), val = tensor([1, 256, 23])]; - tensor var_2337_end_mask_0 = const()[name = tensor("op_2337_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_2337_begin_0, end = var_2337_end_0, end_mask = var_2337_end_mask_0, x = cat)[name = tensor("op_2337")]; - tensor input_323_perm_0 = const()[name = tensor("input_323_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2339 = const()[name = tensor("op_2339"), val = tensor([-1])]; - tensor input_323 = transpose(perm = input_323_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_2340 = reduce_l2_norm(axes = var_2339, keep_dims = var_37, x = input_323)[name = tensor("op_2340")]; - tensor const_28 = const()[name = tensor("const_28"), val = tensor(0x1.fffffep+127)]; - tensor clip_40 = clip(alpha = var_12, beta = const_28, x = var_2340)[name = tensor("clip_40")]; - tensor emb = real_div(x = input_323, y = clip_40)[name = tensor("emb")]; - tensor var_2344_axis_0 = const()[name = tensor("op_2344_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_2344_axis_0, values = (var_215, var_763, var_1311, nkv_1))[name = tensor("op_2344")]; - tensor var_2346_axis_0 = const()[name = tensor("op_2346_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_2346_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_2346")]; - tensor var_2348_axis_0 = const()[name = tensor("op_2348_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_2348_axis_0, values = (input_61, input_141, input_221, input_301))[name = tensor("op_2348")]; - tensor var_2350_axis_0 = const()[name = tensor("op_2350_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_2350_axis_0, values = (clip_9, clip_19, clip_29, clip_39))[name = tensor("op_2350")]; - tensor var_2360 = const()[name = tensor("op_2360"), val = tensor(0x1.5798eep-27)]; - tensor var_2366 = const()[name = tensor("op_2366"), val = tensor(0x1.4f8b58p-17)]; - tensor var_2368 = const()[name = tensor("op_2368"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_2369 = const()[name = tensor("op_2369"), val = tensor(true)]; - tensor var_2371 = const()[name = tensor("op_2371"), val = tensor(0x1p+0)]; - tensor var_2375 = const()[name = tensor("op_2375"), val = tensor(-1)]; - tensor var_2380 = const()[name = tensor("op_2380"), val = tensor(0)]; + tensor var_1477_begin_0 = const()[name = tensor("op_1477_begin_0"), val = tensor([0, 0, 5])]; + tensor var_1477_end_0 = const()[name = tensor("op_1477_end_0"), val = tensor([1, 256, 23])]; + tensor var_1477_end_mask_0 = const()[name = tensor("op_1477_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_1477_begin_0, end = var_1477_end_0, end_mask = var_1477_end_mask_0, x = cat)[name = tensor("op_1477")]; + tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1479 = const()[name = tensor("op_1479"), val = tensor([-1])]; + tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; + tensor var_1480 = reduce_l2_norm(axes = var_1479, keep_dims = var_37, x = input_163)[name = tensor("op_1480")]; + tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; + tensor clip_40 = clip(alpha = var_12, beta = const_12, x = var_1480)[name = tensor("clip_40")]; + tensor emb = real_div(x = input_163, y = clip_40)[name = tensor("emb")]; + tensor var_1484_axis_0 = const()[name = tensor("op_1484_axis_0"), val = tensor(0)]; + tensor enc_kv_new = stack(axis = var_1484_axis_0, values = (var_215, var_548, var_881, nkv_1))[name = tensor("op_1484")]; + tensor var_1486_axis_0 = const()[name = tensor("op_1486_axis_0"), val = tensor(0)]; + tensor enc_scale_new = stack(axis = var_1486_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1486")]; + tensor var_1488_axis_0 = const()[name = tensor("op_1488_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1488_axis_0, values = (window_11, window_23, window_35, window))[name = tensor("op_1488")]; + tensor var_1490_axis_0 = const()[name = tensor("op_1490_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1490_axis_0, values = (clip_9, clip_19, clip_29, clip_39))[name = tensor("op_1490")]; + tensor var_1500 = const()[name = tensor("op_1500"), val = tensor(0x1.5798eep-27)]; + tensor var_1505 = const()[name = tensor("op_1505"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1507 = const()[name = tensor("op_1507"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1508 = const()[name = tensor("op_1508"), val = tensor(true)]; + tensor var_1510 = const()[name = tensor("op_1510"), val = tensor(0x1p+0)]; + tensor var_1514 = const()[name = tensor("op_1514"), val = tensor(-1)]; + tensor var_1520 = const()[name = tensor("op_1520"), val = tensor(0)]; tensor pos = const()[name = tensor("pos"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395840)))]; - tensor var_2442_axes_0 = const()[name = tensor("op_2442_axes_0"), val = tensor([2])]; - tensor var_2442 = expand_dims(axes = var_2442_axes_0, x = emb)[name = tensor("op_2442")]; - tensor var_2443 = const()[name = tensor("op_2443"), val = tensor([1, 1, 12, 1])]; - tensor emb_exp = tile(reps = var_2443, x = var_2442)[name = tensor("emb_exp")]; - tensor input_325_interleave_0 = const()[name = tensor("input_325_interleave_0"), val = tensor(false)]; - tensor input_325 = concat(axis = var_2375, interleave = input_325_interleave_0, values = (emb_exp, pos))[name = tensor("input_325")]; - tensor x_59 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_325)[name = tensor("linear_37")]; - tensor var_2450_perm_0 = const()[name = tensor("op_2450_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2454 = const()[name = tensor("op_2454"), val = tensor([12, 5, 256])]; - tensor var_2450 = transpose(perm = var_2450_perm_0, x = x_59)[name = tensor("transpose_28")]; - tensor x_61 = reshape(shape = var_2454, x = var_2450)[name = tensor("x_61")]; + tensor var_1582_axes_0 = const()[name = tensor("op_1582_axes_0"), val = tensor([2])]; + tensor var_1582 = expand_dims(axes = var_1582_axes_0, x = emb)[name = tensor("op_1582")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 12, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1582)[name = tensor("emb_exp")]; + tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; + tensor input_165 = concat(axis = var_1514, interleave = input_165_interleave_0, values = (emb_exp, pos))[name = tensor("input_165")]; + tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; + tensor var_1590_perm_0 = const()[name = tensor("op_1590_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1594 = const()[name = tensor("op_1594"), val = tensor([12, 5, 256])]; + tensor var_1590 = transpose(perm = var_1590_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1594, x = var_1590)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 12, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -2145,132 +1530,132 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_2462 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_61)[name = tensor("linear_38")]; - tensor var_2463 = const()[name = tensor("op_2463"), val = tensor([12, 5, 4, 64])]; - tensor var_2464 = reshape(shape = var_2463, x = var_2462)[name = tensor("op_2464")]; + tensor var_1602 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1603 = const()[name = tensor("op_1603"), val = tensor([12, 5, 4, 64])]; + tensor var_1604 = reshape(shape = var_1603, x = var_1602)[name = tensor("op_1604")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2468 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_61)[name = tensor("linear_39")]; - tensor var_2469 = const()[name = tensor("op_2469"), val = tensor(0x1p-3)]; - tensor var_2470 = mul(x = var_2468, y = var_2469)[name = tensor("op_2470")]; - tensor var_2471 = const()[name = tensor("op_2471"), val = tensor([12, 5, 4, 64])]; - tensor var_2472 = reshape(shape = var_2471, x = var_2470)[name = tensor("op_2472")]; + tensor var_1608 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1609 = const()[name = tensor("op_1609"), val = tensor(0x1p-3)]; + tensor var_1610 = mul(x = var_1608, y = var_1609)[name = tensor("op_1610")]; + tensor var_1611 = const()[name = tensor("op_1611"), val = tensor([12, 5, 4, 64])]; + tensor var_1612 = reshape(shape = var_1611, x = var_1610)[name = tensor("op_1612")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2476 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_61)[name = tensor("linear_40")]; - tensor var_2477 = const()[name = tensor("op_2477"), val = tensor([12, 5, 4, 64])]; - tensor var_2478 = reshape(shape = var_2477, x = var_2476)[name = tensor("op_2478")]; + tensor var_1616 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1617 = const()[name = tensor("op_1617"), val = tensor([12, 5, 4, 64])]; + tensor var_1618 = reshape(shape = var_1617, x = var_1616)[name = tensor("op_1618")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_329 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_61)[name = tensor("linear_41")]; + tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_2380, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1520, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; - tensor const_36 = const()[name = tensor("const_36"), val = tensor(0x1.fffffep+127)]; - tensor clip_41 = clip(alpha = var_2371, beta = const_36, x = s_t_9)[name = tensor("clip_41")]; + tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; + tensor clip_41 = clip(alpha = var_1510, beta = const_20, x = s_t_9)[name = tensor("clip_41")]; tensor sqrt_s_t_9 = sqrt(x = clip_41)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_2472)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_2464)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1612)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1604)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_2490 = const()[name = tensor("op_2490"), val = tensor([1, 5])]; - tensor var_2491 = reshape(shape = var_2490, x = valid_mask)[name = tensor("op_2491")]; - tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_2491)[name = tensor("causal_with_valid_1")]; - tensor var_2493 = const()[name = tensor("op_2493"), val = tensor([5, 1])]; - tensor var_2494 = reshape(shape = var_2493, x = sqrt_s_t_9)[name = tensor("op_2494")]; - tensor M_9 = real_div(x = causal_with_valid_1, y = var_2494)[name = tensor("M_9")]; - tensor var_2496 = mul(x = qk_9, y = M_9)[name = tensor("op_2496")]; + tensor var_1630 = const()[name = tensor("op_1630"), val = tensor([1, 5])]; + tensor var_1631 = reshape(shape = var_1630, x = valid_mask)[name = tensor("op_1631")]; + tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1631)[name = tensor("causal_with_valid_1")]; + tensor var_1633 = const()[name = tensor("op_1633"), val = tensor([5, 1])]; + tensor var_1634 = reshape(shape = var_1633, x = sqrt_s_t_9)[name = tensor("op_1634")]; + tensor M_9 = real_div(x = causal_with_valid_1, y = var_1634)[name = tensor("M_9")]; + tensor var_1636 = mul(x = qk_9, y = M_9)[name = tensor("op_1636")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_2478)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_2496, y = v_9)[name = tensor("inner_9")]; - tensor var_2498_transpose_x_0 = const()[name = tensor("op_2498_transpose_x_0"), val = tensor(false)]; - tensor var_2498_transpose_y_0 = const()[name = tensor("op_2498_transpose_y_0"), val = tensor(false)]; - tensor var_2498 = matmul(transpose_x = var_2498_transpose_x_0, transpose_y = var_2498_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_2498")]; - tensor var_2499 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_2499")]; - tensor var_2500 = const()[name = tensor("op_2500"), val = tensor([1, 1, 5, 1])]; - tensor var_2501 = reshape(shape = var_2500, x = var_2499)[name = tensor("op_2501")]; - tensor cross_9 = mul(x = var_2498, y = var_2501)[name = tensor("cross_9")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1618)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1636, y = v_9)[name = tensor("inner_9")]; + tensor var_1638_transpose_x_0 = const()[name = tensor("op_1638_transpose_x_0"), val = tensor(false)]; + tensor var_1638_transpose_y_0 = const()[name = tensor("op_1638_transpose_y_0"), val = tensor(false)]; + tensor var_1638 = matmul(transpose_x = var_1638_transpose_x_0, transpose_y = var_1638_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1638")]; + tensor var_1639 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1639")]; + tensor var_1640 = const()[name = tensor("op_1640"), val = tensor([1, 1, 5, 1])]; + tensor var_1641 = reshape(shape = var_1640, x = var_1639)[name = tensor("op_1641")]; + tensor cross_9 = mul(x = var_1638, y = var_1641)[name = tensor("cross_9")]; tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; - tensor var_2504 = const()[name = tensor("op_2504"), val = tensor([1, 1, 5, 1])]; - tensor var_2505 = reshape(shape = var_2504, x = valid_mask)[name = tensor("op_2505")]; - tensor v_masked_1 = mul(x = v_9, y = var_2505)[name = tensor("v_masked_1")]; - tensor var_2507 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_2507")]; - tensor var_2509_transpose_x_1 = const()[name = tensor("op_2509_transpose_x_1"), val = tensor(true)]; - tensor var_2509_transpose_y_1 = const()[name = tensor("op_2509_transpose_y_1"), val = tensor(false)]; - tensor var_2509 = matmul(transpose_x = var_2509_transpose_x_1, transpose_y = var_2509_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_2509")]; - tensor new_kv_unnorm_9 = add(x = var_2507, y = var_2509)[name = tensor("new_kv_unnorm_9")]; - tensor var_2511_keep_dims_0 = const()[name = tensor("op_2511_keep_dims_0"), val = tensor(false)]; - tensor var_2511 = reduce_sum(keep_dims = var_2511_keep_dims_0, x = valid_mask)[name = tensor("op_2511")]; - tensor var_2512 = const()[name = tensor("op_2512"), val = tensor([1])]; - tensor var_2513 = reshape(shape = var_2512, x = var_2511)[name = tensor("op_2513")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_2513)[name = tensor("new_scale_9")]; - tensor const_37 = const()[name = tensor("const_37"), val = tensor(0x1.fffffep+127)]; - tensor clip_42 = clip(alpha = var_2371, beta = const_37, x = new_scale_9)[name = tensor("clip_42")]; + tensor var_1644 = const()[name = tensor("op_1644"), val = tensor([1, 1, 5, 1])]; + tensor var_1645 = reshape(shape = var_1644, x = valid_mask)[name = tensor("op_1645")]; + tensor v_masked_1 = mul(x = v_9, y = var_1645)[name = tensor("v_masked_1")]; + tensor var_1647 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1647")]; + tensor var_1649_transpose_x_1 = const()[name = tensor("op_1649_transpose_x_1"), val = tensor(true)]; + tensor var_1649_transpose_y_1 = const()[name = tensor("op_1649_transpose_y_1"), val = tensor(false)]; + tensor var_1649 = matmul(transpose_x = var_1649_transpose_x_1, transpose_y = var_1649_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1649")]; + tensor new_kv_unnorm_9 = add(x = var_1647, y = var_1649)[name = tensor("new_kv_unnorm_9")]; + tensor var_1651_keep_dims_0 = const()[name = tensor("op_1651_keep_dims_0"), val = tensor(false)]; + tensor var_1651 = reduce_sum(keep_dims = var_1651_keep_dims_0, x = valid_mask)[name = tensor("op_1651")]; + tensor var_1652 = const()[name = tensor("op_1652"), val = tensor([1])]; + tensor var_1653 = reshape(shape = var_1652, x = var_1651)[name = tensor("op_1653")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1653)[name = tensor("new_scale_9")]; + tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; + tensor clip_42 = clip(alpha = var_1510, beta = const_21, x = new_scale_9)[name = tensor("clip_42")]; tensor sqrt_new_scale_1 = sqrt(x = clip_42)[name = tensor("sqrt_new_scale_1")]; - tensor var_2517 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_2517")]; - tensor var_2518_perm_0 = const()[name = tensor("op_2518_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1657 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1657")]; + tensor var_1658_perm_0 = const()[name = tensor("op_1658_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_2518 = transpose(perm = var_2518_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_2368, x = var_2518)[name = tensor("out_27")]; - tensor var_2522 = const()[name = tensor("op_2522"), val = tensor([12, 5, 256])]; - tensor out_29 = reshape(shape = var_2522, x = out_27)[name = tensor("out_29")]; - tensor var_2524 = silu(x = input_329)[name = tensor("op_2524")]; - tensor input_331 = mul(x = var_2524, y = out_29)[name = tensor("input_331")]; - tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_331)[name = tensor("linear_42")]; - tensor input_333 = add(x = x_61, y = ret_out_9)[name = tensor("input_333")]; + tensor var_1658 = transpose(perm = var_1658_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1507, x = var_1658)[name = tensor("out_27")]; + tensor var_1662 = const()[name = tensor("op_1662"), val = tensor([12, 5, 256])]; + tensor out_29 = reshape(shape = var_1662, x = out_27)[name = tensor("out_29")]; + tensor var_1664 = silu(x = input_169)[name = tensor("op_1664")]; + tensor input_171 = mul(x = var_1664, y = out_29)[name = tensor("input_171")]; + tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; + tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_2366, gamma = decoder_norm11_0_weight, x = input_333)[name = tensor("xt_1")]; - tensor var_2534 = const()[name = tensor("op_2534"), val = tensor([1, 12, 5, 256])]; - tensor var_2535 = reshape(shape = var_2534, x = xt_1)[name = tensor("op_2535")]; - tensor var_2536_perm_0 = const()[name = tensor("op_2536_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2539 = const()[name = tensor("op_2539"), val = tensor([5, 12, 256])]; - tensor var_2536 = transpose(perm = var_2536_perm_0, x = var_2535)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_2539, x = var_2536)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1505, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1674 = const()[name = tensor("op_1674"), val = tensor([1, 12, 5, 256])]; + tensor var_1675 = reshape(shape = var_1674, x = xt_1)[name = tensor("op_1675")]; + tensor var_1676_perm_0 = const()[name = tensor("op_1676_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1679 = const()[name = tensor("op_1679"), val = tensor([5, 12, 256])]; + tensor var_1676 = transpose(perm = var_1676_perm_0, x = var_1675)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1679, x = var_1676)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_2562 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_20 = const()[name = tensor("concat_20"), val = tensor([12, 5, 3, 256])]; - tensor var_2564 = reshape(shape = concat_20, x = var_2562)[name = tensor("op_2564")]; - tensor var_2565_axes_0 = const()[name = tensor("op_2565_axes_0"), val = tensor([0])]; - tensor var_2565 = expand_dims(axes = var_2565_axes_0, x = var_2564)[name = tensor("op_2565")]; - tensor var_2566_perm_0 = const()[name = tensor("op_2566_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2567_axes_0 = const()[name = tensor("op_2567_axes_0"), val = tensor([-2])]; - tensor var_2566 = transpose(perm = var_2566_perm_0, x = var_2565)[name = tensor("transpose_21")]; - tensor var_2567 = squeeze(axes = var_2567_axes_0, x = var_2566)[name = tensor("op_2567")]; + tensor var_1702 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_21 = const()[name = tensor("concat_21"), val = tensor([12, 5, 3, 256])]; + tensor var_1704 = reshape(shape = concat_21, x = var_1702)[name = tensor("op_1704")]; + tensor var_1705_axes_0 = const()[name = tensor("op_1705_axes_0"), val = tensor([0])]; + tensor var_1705 = expand_dims(axes = var_1705_axes_0, x = var_1704)[name = tensor("op_1705")]; + tensor var_1706_perm_0 = const()[name = tensor("op_1706_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1707_axes_0 = const()[name = tensor("op_1707_axes_0"), val = tensor([-2])]; + tensor var_1706 = transpose(perm = var_1706_perm_0, x = var_1705)[name = tensor("transpose_21")]; + tensor var_1707 = squeeze(axes = var_1707_axes_0, x = var_1706)[name = tensor("op_1707")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 12, 5, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_2567)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1707)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 12, 5, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_2567)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1707)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 12, 5, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_2567)[name = tensor("v_11")]; - tensor var_2575 = const()[name = tensor("op_2575"), val = tensor([12, 20, 64])]; - tensor var_2576 = reshape(shape = var_2575, x = q_11)[name = tensor("op_2576")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1707)[name = tensor("v_11")]; + tensor var_1715 = const()[name = tensor("op_1715"), val = tensor([12, 20, 64])]; + tensor var_1716 = reshape(shape = var_1715, x = q_11)[name = tensor("op_1716")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2582 = const()[name = tensor("op_2582"), val = tensor([12, 20, 64])]; - tensor var_2583 = reshape(shape = var_2582, x = k_11)[name = tensor("op_2583")]; + tensor var_1722 = const()[name = tensor("op_1722"), val = tensor([12, 20, 64])]; + tensor var_1723 = reshape(shape = var_1722, x = k_11)[name = tensor("op_1723")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2589 = const()[name = tensor("op_2589"), val = tensor([12, 20, 64])]; - tensor var_2590 = reshape(shape = var_2589, x = v_11)[name = tensor("op_2590")]; + tensor var_1729 = const()[name = tensor("op_1729"), val = tensor([12, 20, 64])]; + tensor var_1730 = reshape(shape = var_1729, x = v_11)[name = tensor("op_1730")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2593 = const()[name = tensor("op_2593"), val = tensor([5, 4, 12, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_2576)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_2593, x = q_13)[name = tensor("q_15")]; - tensor var_2595 = const()[name = tensor("op_2595"), val = tensor([5, 4, 12, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_2583)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_2595, x = k_13)[name = tensor("k_15")]; - tensor var_2597 = const()[name = tensor("op_2597"), val = tensor([5, 4, 12, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_2590)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_2597, x = v_13)[name = tensor("v_15")]; + tensor var_1733 = const()[name = tensor("op_1733"), val = tensor([5, 4, 12, 64])]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1716)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1733, x = q_13)[name = tensor("q_15")]; + tensor var_1735 = const()[name = tensor("op_1735"), val = tensor([5, 4, 12, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1723)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1735, x = k_13)[name = tensor("k_15")]; + tensor var_1737 = const()[name = tensor("op_1737"), val = tensor([5, 4, 12, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1730)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1737, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -2281,30 +1666,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_2600 = const()[name = tensor("op_2600"), val = tensor([2, 0, 1, 3])]; - tensor var_2605 = const()[name = tensor("op_2605"), val = tensor([60, 256])]; - tensor var_2601 = transpose(perm = var_2600, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_2605, x = var_2601)[name = tensor("attn_output_3")]; + tensor var_1740 = const()[name = tensor("op_1740"), val = tensor([2, 0, 1, 3])]; + tensor var_1745 = const()[name = tensor("op_1745"), val = tensor([60, 256])]; + tensor var_1741 = transpose(perm = var_1740, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1745, x = var_1741)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_2609 = const()[name = tensor("op_2609"), val = tensor([12, 5, 256])]; - tensor attn_output_7 = reshape(shape = var_2609, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1749 = const()[name = tensor("op_1749"), val = tensor([12, 5, 256])]; + tensor attn_output_7 = reshape(shape = var_1749, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; - tensor input_335 = add(x = query_1, y = sa2_out_1)[name = tensor("input_335")]; - tensor input_337_axes_0 = const()[name = tensor("input_337_axes_0"), val = tensor([-1])]; - tensor input_337 = layer_norm(axes = input_337_axes_0, beta = decoder_norm21_0_bias, epsilon = var_2366, gamma = decoder_norm21_0_weight, x = input_335)[name = tensor("input_337")]; - tensor input_339 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_337)[name = tensor("linear_45")]; - tensor input_341 = relu(x = input_339)[name = tensor("input_341")]; - tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_341)[name = tensor("linear_46")]; - tensor input_343 = add(x = input_337, y = ffn_out_1)[name = tensor("input_343")]; + tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; + tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1505, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; + tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; + tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; + tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_2366, gamma = decoder_norm22_0_weight, x = input_343)[name = tensor("xt_3")]; - tensor var_2629 = const()[name = tensor("op_2629"), val = tensor([1, 5, 12, 256])]; - tensor x_63 = reshape(shape = var_2629, x = xt_3)[name = tensor("x_63")]; - tensor var_2631_perm_0 = const()[name = tensor("op_2631_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2635 = const()[name = tensor("op_2635"), val = tensor([12, 5, 256])]; - tensor var_2631 = transpose(perm = var_2631_perm_0, x = x_63)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_2635, x = var_2631)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1505, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1769 = const()[name = tensor("op_1769"), val = tensor([1, 5, 12, 256])]; + tensor x_31 = reshape(shape = var_1769, x = xt_3)[name = tensor("x_31")]; + tensor var_1771_perm_0 = const()[name = tensor("op_1771_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1775 = const()[name = tensor("op_1775"), val = tensor([12, 5, 256])]; + tensor var_1771 = transpose(perm = var_1771_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1775, x = var_1771)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 12, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -2315,120 +1700,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_2643 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_2644 = const()[name = tensor("op_2644"), val = tensor([12, 5, 4, 64])]; - tensor var_2645 = reshape(shape = var_2644, x = var_2643)[name = tensor("op_2645")]; + tensor var_1783 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1784 = const()[name = tensor("op_1784"), val = tensor([12, 5, 4, 64])]; + tensor var_1785 = reshape(shape = var_1784, x = var_1783)[name = tensor("op_1785")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2649 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_2650 = const()[name = tensor("op_2650"), val = tensor(0x1p-3)]; - tensor var_2651 = mul(x = var_2649, y = var_2650)[name = tensor("op_2651")]; - tensor var_2652 = const()[name = tensor("op_2652"), val = tensor([12, 5, 4, 64])]; - tensor var_2653 = reshape(shape = var_2652, x = var_2651)[name = tensor("op_2653")]; + tensor var_1789 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1790 = const()[name = tensor("op_1790"), val = tensor(0x1p-3)]; + tensor var_1791 = mul(x = var_1789, y = var_1790)[name = tensor("op_1791")]; + tensor var_1792 = const()[name = tensor("op_1792"), val = tensor([12, 5, 4, 64])]; + tensor var_1793 = reshape(shape = var_1792, x = var_1791)[name = tensor("op_1793")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2657 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_2658 = const()[name = tensor("op_2658"), val = tensor([12, 5, 4, 64])]; - tensor var_2659 = reshape(shape = var_2658, x = var_2657)[name = tensor("op_2659")]; + tensor var_1797 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1798 = const()[name = tensor("op_1798"), val = tensor([12, 5, 4, 64])]; + tensor var_1799 = reshape(shape = var_1798, x = var_1797)[name = tensor("op_1799")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_347 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; + tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; - tensor const_48 = const()[name = tensor("const_48"), val = tensor(0x1.fffffep+127)]; - tensor clip_43 = clip(alpha = var_2371, beta = const_48, x = s_t)[name = tensor("clip_43")]; + tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; + tensor clip_43 = clip(alpha = var_1510, beta = const_32, x = s_t)[name = tensor("clip_43")]; tensor sqrt_s_t = sqrt(x = clip_43)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_2653)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_2645)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1793)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1785)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_2674 = const()[name = tensor("op_2674"), val = tensor([5, 1])]; - tensor var_2675 = reshape(shape = var_2674, x = sqrt_s_t)[name = tensor("op_2675")]; - tensor M = real_div(x = causal_with_valid_1, y = var_2675)[name = tensor("M")]; - tensor var_2677 = mul(x = qk, y = M)[name = tensor("op_2677")]; + tensor var_1814 = const()[name = tensor("op_1814"), val = tensor([5, 1])]; + tensor var_1815 = reshape(shape = var_1814, x = sqrt_s_t)[name = tensor("op_1815")]; + tensor M = real_div(x = causal_with_valid_1, y = var_1815)[name = tensor("M")]; + tensor var_1817 = mul(x = qk, y = M)[name = tensor("op_1817")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_2659)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_2677, y = v_17)[name = tensor("inner")]; - tensor var_2679_transpose_x_0 = const()[name = tensor("op_2679_transpose_x_0"), val = tensor(false)]; - tensor var_2679_transpose_y_0 = const()[name = tensor("op_2679_transpose_y_0"), val = tensor(false)]; - tensor var_2679 = matmul(transpose_x = var_2679_transpose_x_0, transpose_y = var_2679_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_2679")]; - tensor var_2680 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_2680")]; - tensor var_2681 = const()[name = tensor("op_2681"), val = tensor([1, 1, 5, 1])]; - tensor var_2682 = reshape(shape = var_2681, x = var_2680)[name = tensor("op_2682")]; - tensor cross = mul(x = var_2679, y = var_2682)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1799)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1817, y = v_17)[name = tensor("inner")]; + tensor var_1819_transpose_x_0 = const()[name = tensor("op_1819_transpose_x_0"), val = tensor(false)]; + tensor var_1819_transpose_y_0 = const()[name = tensor("op_1819_transpose_y_0"), val = tensor(false)]; + tensor var_1819 = matmul(transpose_x = var_1819_transpose_x_0, transpose_y = var_1819_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1819")]; + tensor var_1820 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1820")]; + tensor var_1821 = const()[name = tensor("op_1821"), val = tensor([1, 1, 5, 1])]; + tensor var_1822 = reshape(shape = var_1821, x = var_1820)[name = tensor("op_1822")]; + tensor cross = mul(x = var_1819, y = var_1822)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_2505)[name = tensor("v_masked")]; - tensor var_2688 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_2688")]; - tensor var_2690_transpose_x_1 = const()[name = tensor("op_2690_transpose_x_1"), val = tensor(true)]; - tensor var_2690_transpose_y_1 = const()[name = tensor("op_2690_transpose_y_1"), val = tensor(false)]; - tensor var_2690 = matmul(transpose_x = var_2690_transpose_x_1, transpose_y = var_2690_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_2690")]; - tensor new_kv_unnorm = add(x = var_2688, y = var_2690)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_2513)[name = tensor("new_scale")]; - tensor const_49 = const()[name = tensor("const_49"), val = tensor(0x1.fffffep+127)]; - tensor clip_44 = clip(alpha = var_2371, beta = const_49, x = new_scale)[name = tensor("clip_44")]; + tensor v_masked = mul(x = v_17, y = var_1645)[name = tensor("v_masked")]; + tensor var_1828 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1828")]; + tensor var_1830_transpose_x_1 = const()[name = tensor("op_1830_transpose_x_1"), val = tensor(true)]; + tensor var_1830_transpose_y_1 = const()[name = tensor("op_1830_transpose_y_1"), val = tensor(false)]; + tensor var_1830 = matmul(transpose_x = var_1830_transpose_x_1, transpose_y = var_1830_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1830")]; + tensor new_kv_unnorm = add(x = var_1828, y = var_1830)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1653)[name = tensor("new_scale")]; + tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; + tensor clip_44 = clip(alpha = var_1510, beta = const_33, x = new_scale)[name = tensor("clip_44")]; tensor sqrt_new_scale = sqrt(x = clip_44)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_2699_perm_0 = const()[name = tensor("op_2699_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1839_perm_0 = const()[name = tensor("op_1839_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_2699 = transpose(perm = var_2699_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_2368, x = var_2699)[name = tensor("out_33")]; - tensor var_2703 = const()[name = tensor("op_2703"), val = tensor([12, 5, 256])]; - tensor out = reshape(shape = var_2703, x = out_33)[name = tensor("out")]; - tensor var_2705 = silu(x = input_347)[name = tensor("op_2705")]; - tensor input_349 = mul(x = var_2705, y = out)[name = tensor("input_349")]; - tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_349)[name = tensor("linear_51")]; - tensor input_351 = add(x = x, y = ret_out)[name = tensor("input_351")]; + tensor var_1839 = transpose(perm = var_1839_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1507, x = var_1839)[name = tensor("out_33")]; + tensor var_1843 = const()[name = tensor("op_1843"), val = tensor([12, 5, 256])]; + tensor out = reshape(shape = var_1843, x = out_33)[name = tensor("out")]; + tensor var_1845 = silu(x = input_187)[name = tensor("op_1845")]; + tensor input_189 = mul(x = var_1845, y = out)[name = tensor("input_189")]; + tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; + tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_2366, gamma = decoder_norm11_1_weight, x = input_351)[name = tensor("xt_5")]; - tensor var_2715 = const()[name = tensor("op_2715"), val = tensor([1, 12, 5, 256])]; - tensor var_2716 = reshape(shape = var_2715, x = xt_5)[name = tensor("op_2716")]; - tensor var_2717_perm_0 = const()[name = tensor("op_2717_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2720 = const()[name = tensor("op_2720"), val = tensor([5, 12, 256])]; - tensor var_2717 = transpose(perm = var_2717_perm_0, x = var_2716)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_2720, x = var_2717)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1505, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1855 = const()[name = tensor("op_1855"), val = tensor([1, 12, 5, 256])]; + tensor var_1856 = reshape(shape = var_1855, x = xt_5)[name = tensor("op_1856")]; + tensor var_1857_perm_0 = const()[name = tensor("op_1857_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1860 = const()[name = tensor("op_1860"), val = tensor([5, 12, 256])]; + tensor var_1857 = transpose(perm = var_1857_perm_0, x = var_1856)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1860, x = var_1857)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_2743 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_21 = const()[name = tensor("concat_21"), val = tensor([12, 5, 3, 256])]; - tensor var_2745 = reshape(shape = concat_21, x = var_2743)[name = tensor("op_2745")]; - tensor var_2746_axes_0 = const()[name = tensor("op_2746_axes_0"), val = tensor([0])]; - tensor var_2746 = expand_dims(axes = var_2746_axes_0, x = var_2745)[name = tensor("op_2746")]; - tensor var_2747_perm_0 = const()[name = tensor("op_2747_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2748_axes_0 = const()[name = tensor("op_2748_axes_0"), val = tensor([-2])]; - tensor var_2747 = transpose(perm = var_2747_perm_0, x = var_2746)[name = tensor("transpose_8")]; - tensor var_2748 = squeeze(axes = var_2748_axes_0, x = var_2747)[name = tensor("op_2748")]; + tensor var_1883 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_22 = const()[name = tensor("concat_22"), val = tensor([12, 5, 3, 256])]; + tensor var_1885 = reshape(shape = concat_22, x = var_1883)[name = tensor("op_1885")]; + tensor var_1886_axes_0 = const()[name = tensor("op_1886_axes_0"), val = tensor([0])]; + tensor var_1886 = expand_dims(axes = var_1886_axes_0, x = var_1885)[name = tensor("op_1886")]; + tensor var_1887_perm_0 = const()[name = tensor("op_1887_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1888_axes_0 = const()[name = tensor("op_1888_axes_0"), val = tensor([-2])]; + tensor var_1887 = transpose(perm = var_1887_perm_0, x = var_1886)[name = tensor("transpose_8")]; + tensor var_1888 = squeeze(axes = var_1888_axes_0, x = var_1887)[name = tensor("op_1888")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 12, 5, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_2748)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1888)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 12, 5, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_2748)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1888)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 12, 5, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_2748)[name = tensor("v_19")]; - tensor var_2756 = const()[name = tensor("op_2756"), val = tensor([12, 20, 64])]; - tensor var_2757 = reshape(shape = var_2756, x = q_19)[name = tensor("op_2757")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1888)[name = tensor("v_19")]; + tensor var_1896 = const()[name = tensor("op_1896"), val = tensor([12, 20, 64])]; + tensor var_1897 = reshape(shape = var_1896, x = q_19)[name = tensor("op_1897")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2763 = const()[name = tensor("op_2763"), val = tensor([12, 20, 64])]; - tensor var_2764 = reshape(shape = var_2763, x = k_19)[name = tensor("op_2764")]; + tensor var_1903 = const()[name = tensor("op_1903"), val = tensor([12, 20, 64])]; + tensor var_1904 = reshape(shape = var_1903, x = k_19)[name = tensor("op_1904")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2770 = const()[name = tensor("op_2770"), val = tensor([12, 20, 64])]; - tensor var_2771 = reshape(shape = var_2770, x = v_19)[name = tensor("op_2771")]; + tensor var_1910 = const()[name = tensor("op_1910"), val = tensor([12, 20, 64])]; + tensor var_1911 = reshape(shape = var_1910, x = v_19)[name = tensor("op_1911")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2774 = const()[name = tensor("op_2774"), val = tensor([5, 4, 12, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_2757)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_2774, x = q_21)[name = tensor("q")]; - tensor var_2776 = const()[name = tensor("op_2776"), val = tensor([5, 4, 12, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_2764)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_2776, x = k_21)[name = tensor("k")]; - tensor var_2778 = const()[name = tensor("op_2778"), val = tensor([5, 4, 12, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_2771)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_2778, x = v_21)[name = tensor("v")]; + tensor var_1914 = const()[name = tensor("op_1914"), val = tensor([5, 4, 12, 64])]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1897)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1914, x = q_21)[name = tensor("q")]; + tensor var_1916 = const()[name = tensor("op_1916"), val = tensor([5, 4, 12, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1904)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1916, x = k_21)[name = tensor("k")]; + tensor var_1918 = const()[name = tensor("op_1918"), val = tensor([5, 4, 12, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1911)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1918, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -2439,50 +1824,50 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_2781 = const()[name = tensor("op_2781"), val = tensor([2, 0, 1, 3])]; - tensor var_2786 = const()[name = tensor("op_2786"), val = tensor([60, 256])]; - tensor var_2782 = transpose(perm = var_2781, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_2786, x = var_2782)[name = tensor("attn_output_11")]; + tensor var_1921 = const()[name = tensor("op_1921"), val = tensor([2, 0, 1, 3])]; + tensor var_1926 = const()[name = tensor("op_1926"), val = tensor([60, 256])]; + tensor var_1922 = transpose(perm = var_1921, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1926, x = var_1922)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_2790 = const()[name = tensor("op_2790"), val = tensor([12, 5, 256])]; - tensor attn_output = reshape(shape = var_2790, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1930 = const()[name = tensor("op_1930"), val = tensor([12, 5, 256])]; + tensor attn_output = reshape(shape = var_1930, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; - tensor input_353 = add(x = query_5, y = sa2_out)[name = tensor("input_353")]; - tensor input_355_axes_0 = const()[name = tensor("input_355_axes_0"), val = tensor([-1])]; - tensor input_355 = layer_norm(axes = input_355_axes_0, beta = decoder_norm21_1_bias, epsilon = var_2366, gamma = decoder_norm21_1_weight, x = input_353)[name = tensor("input_355")]; - tensor input_357 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_355)[name = tensor("linear_54")]; - tensor input_359 = relu(x = input_357)[name = tensor("input_359")]; - tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_359)[name = tensor("linear_55")]; - tensor input_361 = add(x = input_355, y = ffn_out)[name = tensor("input_361")]; + tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; + tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1505, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; + tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; + tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; + tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_2366, gamma = decoder_norm22_1_weight, x = input_361)[name = tensor("xt")]; - tensor var_2810 = const()[name = tensor("op_2810"), val = tensor([1, 5, 12, 256])]; - tensor input = reshape(shape = var_2810, x = xt)[name = tensor("input")]; - tensor var_2812 = const()[name = tensor("op_2812"), val = tensor([-1])]; - tensor var_2813 = reduce_l2_norm(axes = var_2812, keep_dims = var_2369, x = input)[name = tensor("op_2813")]; - tensor const_58 = const()[name = tensor("const_58"), val = tensor(0x1.fffffep+127)]; - tensor clip_45 = clip(alpha = var_2360, beta = const_58, x = var_2813)[name = tensor("clip_45")]; - tensor var_2815 = real_div(x = input, y = clip_45)[name = tensor("op_2815")]; - tensor concat_25 = const()[name = tensor("concat_25"), val = tensor([5, 1, 256])]; - tensor reshape_0 = reshape(shape = concat_25, x = emb)[name = tensor("reshape_0")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1505, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1950 = const()[name = tensor("op_1950"), val = tensor([1, 5, 12, 256])]; + tensor input = reshape(shape = var_1950, x = xt)[name = tensor("input")]; + tensor var_1952 = const()[name = tensor("op_1952"), val = tensor([-1])]; + tensor var_1953 = reduce_l2_norm(axes = var_1952, keep_dims = var_1508, x = input)[name = tensor("op_1953")]; + tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; + tensor clip_45 = clip(alpha = var_1500, beta = const_42, x = var_1953)[name = tensor("clip_45")]; + tensor var_1955 = real_div(x = input, y = clip_45)[name = tensor("op_1955")]; + tensor concat_26 = const()[name = tensor("concat_26"), val = tensor([5, 1, 256])]; + tensor reshape_0 = reshape(shape = concat_26, x = emb)[name = tensor("reshape_0")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_26 = const()[name = tensor("concat_26"), val = tensor([5, 256, 12])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_2815)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_26, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_27 = const()[name = tensor("concat_27"), val = tensor([5, 256, 12])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1955)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_27, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = reshape_0, y = reshape_1)[name = tensor("matmul_2")]; - tensor concat_29 = const()[name = tensor("concat_29"), val = tensor([1, 5, 12])]; - tensor reshape_2 = reshape(shape = concat_29, x = matmul_2)[name = tensor("reshape_2")]; + tensor concat_30 = const()[name = tensor("concat_30"), val = tensor([1, 5, 12])]; + tensor reshape_2 = reshape(shape = concat_30, x = matmul_2)[name = tensor("reshape_2")]; tensor output_begin_0 = const()[name = tensor("output_begin_0"), val = tensor([0, 0, 1])]; tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 5, 11])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = reshape_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_2819")]; - tensor var_2821_axis_0 = const()[name = tensor("op_2821_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_2821_axis_0, values = (var_2517, nkv))[name = tensor("op_2821")]; - tensor var_2823_axis_0 = const()[name = tensor("op_2823_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_2823_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_2823")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1959")]; + tensor var_1961_axis_0 = const()[name = tensor("op_1961_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1961_axis_0, values = (var_1657, nkv))[name = tensor("op_1961")]; + tensor var_1963_axis_0 = const()[name = tensor("op_1963_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1963_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1963")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/DIHARD II/ls_eend_dih2_500ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/DIHARD II/ls_eend_dih2_500ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index 68574f7c0ecc8813f8bbe1bcf86d2e88a2dceaf9..226dbba9b89f99ba80e447da33e47d03ac3f77f0 100644 --- a/optimized/DIHARD II/ls_eend_dih2_500ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/DIHARD II/ls_eend_dih2_500ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cc644df366f82403d73b65f23bef51b1ca56d9fb8e19efabfc35370577637030 -size 352369 +oid sha256:09bfcd09aa0dbfee564cf3e2e096c5f973a952fe6f48fb2adad39f07cc32c8ac +size 255679 diff --git a/optimized/DIHARD II/ls_eend_dih2_500ms.mlpackage/Manifest.json b/optimized/DIHARD II/ls_eend_dih2_500ms.mlpackage/Manifest.json index 2881f09fc5532e83701f89330d3f582486be3d67..fa081c6ed7962be706e91c56aea4a517ae366985 100644 --- a/optimized/DIHARD II/ls_eend_dih2_500ms.mlpackage/Manifest.json +++ b/optimized/DIHARD II/ls_eend_dih2_500ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "6EA79E29-81AF-4C5D-A21D-5C9089141F7A": { - "author": "com.apple.CoreML", - "description": "CoreML Model Specification", - "name": "model.mlmodel", - "path": "com.apple.CoreML/model.mlmodel" - }, - "972E217A-A634-4C26-8BA3-FB540E485FBD": { + "C222E77B-3C93-43F7-A184-786CE7D76148": { "author": "com.apple.CoreML", "description": "CoreML Model Weights", "name": "weights", "path": "com.apple.CoreML/weights" + }, + "D3EB2F6B-E9A8-4E68-AFEA-BF6BD81702D3": { + "author": "com.apple.CoreML", + "description": "CoreML Model Specification", + "name": "model.mlmodel", + "path": "com.apple.CoreML/model.mlmodel" } }, - "rootModelIdentifier": "6EA79E29-81AF-4C5D-A21D-5C9089141F7A" + "rootModelIdentifier": "D3EB2F6B-E9A8-4E68-AFEA-BF6BD81702D3" } diff --git a/optimized/DIHARD III/.DS_Store b/optimized/DIHARD III/.DS_Store index b67321785be2a47dbf01f3e158834fca166f50e1..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 100644 Binary files a/optimized/DIHARD III/.DS_Store and b/optimized/DIHARD III/.DS_Store differ diff --git a/optimized/DIHARD III/ls_eend_dih3_100ms.mlmodelc/analytics/coremldata.bin b/optimized/DIHARD III/ls_eend_dih3_100ms.mlmodelc/analytics/coremldata.bin index 62b0a098cb6c8fc532236348f6c387249fd98a9a..f28e6d822bbb54e9d452daa9c4a41fa5ecb33def 100644 --- a/optimized/DIHARD III/ls_eend_dih3_100ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/DIHARD III/ls_eend_dih3_100ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:174b06531f88199dbfdd0bc7086d2739e045a95fdbf6e3bcd45d025931638ba8 +oid sha256:33ee0ad8ef8dc4f7723c0b04e3cb187293d0313a078036f4d875f45c320302ac size 243 diff --git a/optimized/DIHARD III/ls_eend_dih3_100ms.mlmodelc/coremldata.bin b/optimized/DIHARD III/ls_eend_dih3_100ms.mlmodelc/coremldata.bin index a59476907a24b26318cda68329810d8c19192d6a..f6bdc00b88c75562ed0dd2c4fb9dc58314bbbddb 100644 --- a/optimized/DIHARD III/ls_eend_dih3_100ms.mlmodelc/coremldata.bin +++ b/optimized/DIHARD III/ls_eend_dih3_100ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:df7fd8a0a7a2a126e498115f3102212c7741ac59db6fa488e29762642000e68d -size 1268 +oid sha256:6577a5b5c068ad6d933e242f4f830adf1126ed8805aa57d50510d90b6fc49d51 +size 1359 diff --git a/optimized/DIHARD III/ls_eend_dih3_100ms.mlmodelc/metadata.json b/optimized/DIHARD III/ls_eend_dih3_100ms.mlmodelc/metadata.json index 47dacdfa5839739e07e7a0035dba9ae5189f3d0f..a6a0b21bcf7cc8bb4c9c6593503e807dbf502ed2 100644 --- a/optimized/DIHARD III/ls_eend_dih3_100ms.mlmodelc/metadata.json +++ b/optimized/DIHARD III/ls_eend_dih3_100ms.mlmodelc/metadata.json @@ -96,7 +96,7 @@ "Ios17.reshape" : 74, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 53, + "Ios17.transpose" : 57, "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 58, @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"dih3\", \"model_label\": \"DIHARD III\", \"variant\": \"pipeline\", \"T\": 1, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"dih3\", \"model_label\": \"DIHARD III\", \"variant\": \"pipeline\", \"chunk_size\": 1, \"step_duration_ms\": 100, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/DIHARD III/ls_eend_dih3_100ms.mlmodelc/model.mil b/optimized/DIHARD III/ls_eend_dih3_100ms.mlmodelc/model.mil index 742a1c536563c04e1f971f8b28aa14c6f74c6dfa..a20414bfdbdabb07fc1d4dae565285b934f5f3a5 100644 --- a/optimized/DIHARD III/ls_eend_dih3_100ms.mlmodelc/model.mil +++ b/optimized/DIHARD III/ls_eend_dih3_100ms.mlmodelc/model.mil @@ -216,6 +216,7 @@ program(1.0) tensor var_21 = const()[name = tensor("op_21"), val = tensor(0x0p+0)]; tensor var_24 = const()[name = tensor("op_24"), val = tensor(0x1.0c6f7ap-20)]; tensor var_27 = const()[name = tensor("op_27"), val = tensor(2)]; + tensor var_30 = const()[name = tensor("op_30"), val = tensor(0)]; tensor var_33 = const()[name = tensor("op_33"), val = tensor(1)]; tensor var_35 = const()[name = tensor("op_35"), val = tensor(0x1.4f8b58p-17)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(true)]; @@ -262,8 +263,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_53")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_54")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_196 = const()[name = tensor("op_196"), val = tensor([1, 1])]; tensor var_197 = reshape(shape = var_196, x = sqrt_s_t_1)[name = tensor("op_197")]; @@ -271,7 +272,7 @@ program(1.0) tensor var_199 = mul(x = qk_1, y = M_1)[name = tensor("op_199")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_52")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_199, y = v_1)[name = tensor("inner_1")]; tensor var_201_transpose_x_0 = const()[name = tensor("op_201_transpose_x_0"), val = tensor(false)]; tensor var_201_transpose_y_0 = const()[name = tensor("op_201_transpose_y_0"), val = tensor(false)]; @@ -292,7 +293,7 @@ program(1.0) tensor var_214 = real_div(x = new_kv_unnorm_1, y = var_213)[name = tensor("op_214")]; tensor var_215_perm_0 = const()[name = tensor("op_215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_51")]; + tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_24, x = var_215)[name = tensor("out_3")]; tensor var_219 = const()[name = tensor("op_219"), val = tensor([1, 1, 256])]; tensor out_5 = reshape(shape = var_219, x = out_3)[name = tensor("out_5")]; @@ -336,10 +337,12 @@ program(1.0) tensor var_252 = sub(x = var_17, y = use_roll_1)[name = tensor("op_252")]; tensor var_253 = mul(x = filled_window_1, y = var_252)[name = tensor("op_253")]; tensor var_254 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_254")]; - tensor input_21 = add(x = var_253, y = var_254)[name = tensor("input_21")]; + tensor window_3 = add(x = var_253, y = var_254)[name = tensor("window_3")]; tensor var_256 = const()[name = tensor("op_256"), val = tensor(0x1p+0)]; tensor var_257 = add(x = filled_1, y = var_256)[name = tensor("op_257")]; tensor clip_1 = clip(alpha = var_21, beta = var_16, x = var_257)[name = tensor("clip_1")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_30, interleave = input_21_interleave_0, values = window_3)[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; @@ -348,13 +351,13 @@ program(1.0) tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_50")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_278_split_sizes_0 = const()[name = tensor("op_278_split_sizes_0"), val = tensor([256, 256])]; - tensor var_278_axis_0 = const()[name = tensor("op_278_axis_0"), val = tensor(1)]; - tensor var_278_0, tensor var_278_1 = split(axis = var_278_axis_0, split_sizes = var_278_split_sizes_0, x = inputs_3)[name = tensor("op_278")]; - tensor var_280 = sigmoid(x = var_278_1)[name = tensor("op_280")]; - tensor inputs_5 = mul(x = var_278_0, y = var_280)[name = tensor("inputs_5")]; + tensor var_280_split_sizes_0 = const()[name = tensor("op_280_split_sizes_0"), val = tensor([256, 256])]; + tensor var_280_axis_0 = const()[name = tensor("op_280_axis_0"), val = tensor(1)]; + tensor var_280_0, tensor var_280_1 = split(axis = var_280_axis_0, split_sizes = var_280_split_sizes_0, x = inputs_3)[name = tensor("op_280")]; + tensor var_282 = sigmoid(x = var_280_1)[name = tensor("op_282")]; + tensor inputs_5 = mul(x = var_280_0, y = var_282)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; @@ -374,22 +377,22 @@ program(1.0) tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_311_begin_0 = const()[name = tensor("op_311_begin_0"), val = tensor([0, -1, 0])]; - tensor var_311_end_0 = const()[name = tensor("op_311_end_0"), val = tensor([1, 16, 256])]; - tensor var_311_end_mask_0 = const()[name = tensor("op_311_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_49")]; - tensor var_311 = slice_by_index(begin = var_311_begin_0, end = var_311_end_0, end_mask = var_311_end_mask_0, x = conv_out_1)[name = tensor("op_311")]; - tensor var_314_interleave_0 = const()[name = tensor("op_314_interleave_0"), val = tensor(false)]; - tensor var_314 = concat(axis = var_33, interleave = var_314_interleave_0, values = var_311)[name = tensor("op_314")]; - tensor input_31 = add(x = x_3, y = var_314)[name = tensor("input_31")]; + tensor var_313_begin_0 = const()[name = tensor("op_313_begin_0"), val = tensor([0, -1, 0])]; + tensor var_313_end_0 = const()[name = tensor("op_313_end_0"), val = tensor([1, 16, 256])]; + tensor var_313_end_mask_0 = const()[name = tensor("op_313_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_313 = slice_by_index(begin = var_313_begin_0, end = var_313_end_0, end_mask = var_313_end_mask_0, x = conv_out_1)[name = tensor("op_313")]; + tensor var_315_perm_0 = const()[name = tensor("op_315_perm_0"), val = tensor([1, 0, 2])]; + tensor var_315 = transpose(perm = var_315_perm_0, x = var_313)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_315)[name = tensor("input_31")]; tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; - tensor var_337 = const()[name = tensor("op_337"), val = tensor(0x1p-1)]; - tensor var_338 = mul(x = input_39, y = var_337)[name = tensor("op_338")]; - tensor input_41 = add(x = var_338, y = input_31)[name = tensor("input_41")]; + tensor var_338 = const()[name = tensor("op_338"), val = tensor(0x1p-1)]; + tensor var_339 = mul(x = input_39, y = var_338)[name = tensor("op_339")]; + tensor input_41 = add(x = var_339, y = input_31)[name = tensor("input_41")]; tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; @@ -397,9 +400,9 @@ program(1.0) tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; - tensor var_367 = const()[name = tensor("op_367"), val = tensor(0x1p-1)]; - tensor var_368 = mul(x = input_51, y = var_367)[name = tensor("op_368")]; - tensor input_53 = add(x = var_368, y = input_43)[name = tensor("input_53")]; + tensor var_368 = const()[name = tensor("op_368"), val = tensor(0x1p-1)]; + tensor var_369 = mul(x = input_51, y = var_368)[name = tensor("op_369")]; + tensor input_53 = add(x = var_369, y = input_43)[name = tensor("input_53")]; tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; @@ -412,19 +415,19 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_382 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; - tensor var_383 = const()[name = tensor("op_383"), val = tensor([1, 1, 4, 64])]; - tensor var_384 = reshape(shape = var_383, x = var_382)[name = tensor("op_384")]; + tensor var_383 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_384 = const()[name = tensor("op_384"), val = tensor([1, 1, 4, 64])]; + tensor var_385 = reshape(shape = var_384, x = var_383)[name = tensor("op_385")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_388 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; - tensor var_389 = const()[name = tensor("op_389"), val = tensor(0x1p-3)]; - tensor var_390 = mul(x = var_388, y = var_389)[name = tensor("op_390")]; - tensor var_391 = const()[name = tensor("op_391"), val = tensor([1, 1, 4, 64])]; - tensor var_392 = reshape(shape = var_391, x = var_390)[name = tensor("op_392")]; + tensor var_389 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_390 = const()[name = tensor("op_390"), val = tensor(0x1p-3)]; + tensor var_391 = mul(x = var_389, y = var_390)[name = tensor("op_391")]; + tensor var_392 = const()[name = tensor("op_392"), val = tensor([1, 1, 4, 64])]; + tensor var_393 = reshape(shape = var_392, x = var_391)[name = tensor("op_393")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_396 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; - tensor var_397 = const()[name = tensor("op_397"), val = tensor([1, 1, 4, 64])]; - tensor var_398 = reshape(shape = var_397, x = var_396)[name = tensor("op_398")]; + tensor var_397 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_398 = const()[name = tensor("op_398"), val = tensor([1, 1, 4, 64])]; + tensor var_399 = reshape(shape = var_398, x = var_397)[name = tensor("op_399")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; @@ -432,84 +435,86 @@ program(1.0) tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_392)[name = tensor("transpose_47")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_384)[name = tensor("transpose_48")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_393)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_385)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_408 = const()[name = tensor("op_408"), val = tensor([1, 1])]; - tensor var_409 = reshape(shape = var_408, x = sqrt_s_t_3)[name = tensor("op_409")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_409)[name = tensor("M_3")]; - tensor var_411 = mul(x = qk_3, y = M_3)[name = tensor("op_411")]; + tensor var_409 = const()[name = tensor("op_409"), val = tensor([1, 1])]; + tensor var_410 = reshape(shape = var_409, x = sqrt_s_t_3)[name = tensor("op_410")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_410)[name = tensor("M_3")]; + tensor var_412 = mul(x = qk_3, y = M_3)[name = tensor("op_412")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_398)[name = tensor("transpose_46")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_411, y = v_3)[name = tensor("inner_3")]; - tensor var_413_transpose_x_0 = const()[name = tensor("op_413_transpose_x_0"), val = tensor(false)]; - tensor var_413_transpose_y_0 = const()[name = tensor("op_413_transpose_y_0"), val = tensor(false)]; - tensor var_413 = matmul(transpose_x = var_413_transpose_x_0, transpose_y = var_413_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_413")]; - tensor var_414 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_414")]; - tensor var_415 = const()[name = tensor("op_415"), val = tensor([1, 1, 1, 1])]; - tensor var_416 = reshape(shape = var_415, x = var_414)[name = tensor("op_416")]; - tensor cross_3 = mul(x = var_413, y = var_416)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_399)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_412, y = v_3)[name = tensor("inner_3")]; + tensor var_414_transpose_x_0 = const()[name = tensor("op_414_transpose_x_0"), val = tensor(false)]; + tensor var_414_transpose_y_0 = const()[name = tensor("op_414_transpose_y_0"), val = tensor(false)]; + tensor var_414 = matmul(transpose_x = var_414_transpose_x_0, transpose_y = var_414_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_414")]; + tensor var_415 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_415")]; + tensor var_416 = const()[name = tensor("op_416"), val = tensor([1, 1, 1, 1])]; + tensor var_417 = reshape(shape = var_416, x = var_415)[name = tensor("op_417")]; + tensor cross_3 = mul(x = var_414, y = var_417)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_419 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_419")]; - tensor var_421_transpose_x_1 = const()[name = tensor("op_421_transpose_x_1"), val = tensor(true)]; - tensor var_421_transpose_y_1 = const()[name = tensor("op_421_transpose_y_1"), val = tensor(false)]; - tensor var_421 = matmul(transpose_x = var_421_transpose_x_1, transpose_y = var_421_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_421")]; - tensor new_kv_unnorm_3 = add(x = var_419, y = var_421)[name = tensor("new_kv_unnorm_3")]; - tensor var_423 = const()[name = tensor("op_423"), val = tensor(0x1p+0)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_423)[name = tensor("new_scale_3")]; - tensor var_425 = sqrt(x = new_scale_3)[name = tensor("op_425")]; - tensor var_426 = real_div(x = new_kv_unnorm_3, y = var_425)[name = tensor("op_426")]; - tensor var_427_perm_0 = const()[name = tensor("op_427_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_420 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_420")]; + tensor var_422_transpose_x_1 = const()[name = tensor("op_422_transpose_x_1"), val = tensor(true)]; + tensor var_422_transpose_y_1 = const()[name = tensor("op_422_transpose_y_1"), val = tensor(false)]; + tensor var_422 = matmul(transpose_x = var_422_transpose_x_1, transpose_y = var_422_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_422")]; + tensor new_kv_unnorm_3 = add(x = var_420, y = var_422)[name = tensor("new_kv_unnorm_3")]; + tensor var_424 = const()[name = tensor("op_424"), val = tensor(0x1p+0)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_424)[name = tensor("new_scale_3")]; + tensor var_426 = sqrt(x = new_scale_3)[name = tensor("op_426")]; + tensor var_427 = real_div(x = new_kv_unnorm_3, y = var_426)[name = tensor("op_427")]; + tensor var_428_perm_0 = const()[name = tensor("op_428_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_427 = transpose(perm = var_427_perm_0, x = out_7)[name = tensor("transpose_45")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_427)[name = tensor("out_9")]; - tensor var_431 = const()[name = tensor("op_431"), val = tensor([1, 1, 256])]; - tensor out_11 = reshape(shape = var_431, x = out_9)[name = tensor("out_11")]; - tensor var_433 = silu(x = input_57)[name = tensor("op_433")]; - tensor input_59 = mul(x = var_433, y = out_11)[name = tensor("input_59")]; + tensor var_428 = transpose(perm = var_428_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_428)[name = tensor("out_9")]; + tensor var_432 = const()[name = tensor("op_432"), val = tensor([1, 1, 256])]; + tensor out_11 = reshape(shape = var_432, x = out_9)[name = tensor("out_11")]; + tensor var_434 = silu(x = input_57)[name = tensor("op_434")]; + tensor input_59 = mul(x = var_434, y = out_11)[name = tensor("input_59")]; tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; tensor filled_3_begin_0 = const()[name = tensor("filled_3_begin_0"), val = tensor([1, 0])]; tensor filled_3_end_0 = const()[name = tensor("filled_3_end_0"), val = tensor([2, 1])]; tensor filled_3_end_mask_0 = const()[name = tensor("filled_3_end_mask_0"), val = tensor([false, true])]; tensor filled_3_squeeze_mask_0 = const()[name = tensor("filled_3_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_3 = slice_by_index(begin = filled_3_begin_0, end = filled_3_end_0, end_mask = filled_3_end_mask_0, squeeze_mask = filled_3_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_3")]; - tensor var_445_begin_0 = const()[name = tensor("op_445_begin_0"), val = tensor([0, 1, 0])]; - tensor var_445_end_0 = const()[name = tensor("op_445_end_0"), val = tensor([1, 16, 256])]; - tensor var_445_end_mask_0 = const()[name = tensor("op_445_end_mask_0"), val = tensor([true, true, true])]; - tensor var_445 = slice_by_index(begin = var_445_begin_0, end = var_445_end_0, end_mask = var_445_end_mask_0, x = window_3)[name = tensor("op_445")]; + tensor var_446_begin_0 = const()[name = tensor("op_446_begin_0"), val = tensor([0, 1, 0])]; + tensor var_446_end_0 = const()[name = tensor("op_446_end_0"), val = tensor([1, 16, 256])]; + tensor var_446_end_mask_0 = const()[name = tensor("op_446_end_mask_0"), val = tensor([true, true, true])]; + tensor var_446 = slice_by_index(begin = var_446_begin_0, end = var_446_end_0, end_mask = var_446_end_mask_0, x = window_5)[name = tensor("op_446")]; tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_445, x_9))[name = tensor("rolled_3")]; + tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_446, x_9))[name = tensor("rolled_3")]; tensor clip_2 = clip(alpha = var_21, beta = var_20, x = filled_3)[name = tensor("clip_2")]; - tensor var_450 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_450")]; - tensor var_451_dtype_0 = const()[name = tensor("op_451_dtype_0"), val = tensor("fp32")]; - tensor var_452 = const()[name = tensor("op_452"), val = tensor([1, 16, 1])]; - tensor var_451 = cast(dtype = var_451_dtype_0, x = var_450)[name = tensor("cast_125")]; - tensor fill_mask_3 = reshape(shape = var_452, x = var_451)[name = tensor("fill_mask_3")]; + tensor var_451 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_451")]; + tensor var_452_dtype_0 = const()[name = tensor("op_452_dtype_0"), val = tensor("fp32")]; + tensor var_453 = const()[name = tensor("op_453"), val = tensor([1, 16, 1])]; + tensor var_452 = cast(dtype = var_452_dtype_0, x = var_451)[name = tensor("cast_125")]; + tensor fill_mask_3 = reshape(shape = var_453, x = var_452)[name = tensor("fill_mask_3")]; tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = x_9)[name = tensor("x_expanded_3")]; - tensor var_456 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_456")]; - tensor var_457 = mul(x = window_3, y = var_456)[name = tensor("op_457")]; - tensor var_458 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_458")]; - tensor filled_window_3 = add(x = var_457, y = var_458)[name = tensor("filled_window_3")]; - tensor var_460 = greater_equal(x = filled_3, y = var_16)[name = tensor("op_460")]; - tensor var_461_dtype_0 = const()[name = tensor("op_461_dtype_0"), val = tensor("fp32")]; - tensor var_462 = const()[name = tensor("op_462"), val = tensor([1, 1, 1])]; - tensor var_461 = cast(dtype = var_461_dtype_0, x = var_460)[name = tensor("cast_124")]; - tensor use_roll_3 = reshape(shape = var_462, x = var_461)[name = tensor("use_roll_3")]; - tensor var_464 = sub(x = var_17, y = use_roll_3)[name = tensor("op_464")]; - tensor var_465 = mul(x = filled_window_3, y = var_464)[name = tensor("op_465")]; - tensor var_466 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_466")]; - tensor input_61 = add(x = var_465, y = var_466)[name = tensor("input_61")]; - tensor var_468 = const()[name = tensor("op_468"), val = tensor(0x1p+0)]; - tensor var_469 = add(x = filled_3, y = var_468)[name = tensor("op_469")]; - tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_469)[name = tensor("clip_3")]; + tensor var_457 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_457")]; + tensor var_458 = mul(x = window_5, y = var_457)[name = tensor("op_458")]; + tensor var_459 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_459")]; + tensor filled_window_3 = add(x = var_458, y = var_459)[name = tensor("filled_window_3")]; + tensor var_461 = greater_equal(x = filled_3, y = var_16)[name = tensor("op_461")]; + tensor var_462_dtype_0 = const()[name = tensor("op_462_dtype_0"), val = tensor("fp32")]; + tensor var_463 = const()[name = tensor("op_463"), val = tensor([1, 1, 1])]; + tensor var_462 = cast(dtype = var_462_dtype_0, x = var_461)[name = tensor("cast_124")]; + tensor use_roll_3 = reshape(shape = var_463, x = var_462)[name = tensor("use_roll_3")]; + tensor var_465 = sub(x = var_17, y = use_roll_3)[name = tensor("op_465")]; + tensor var_466 = mul(x = filled_window_3, y = var_465)[name = tensor("op_466")]; + tensor var_467 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_467")]; + tensor window_7 = add(x = var_466, y = var_467)[name = tensor("window_7")]; + tensor var_469 = const()[name = tensor("op_469"), val = tensor(0x1p+0)]; + tensor var_470 = add(x = filled_3, y = var_469)[name = tensor("op_470")]; + tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_470)[name = tensor("clip_3")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_30, interleave = input_61_interleave_0, values = window_7)[name = tensor("input_61")]; tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; @@ -518,13 +523,13 @@ program(1.0) tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; - tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_44")]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; - tensor var_490_split_sizes_0 = const()[name = tensor("op_490_split_sizes_0"), val = tensor([256, 256])]; - tensor var_490_axis_0 = const()[name = tensor("op_490_axis_0"), val = tensor(1)]; - tensor var_490_0, tensor var_490_1 = split(axis = var_490_axis_0, split_sizes = var_490_split_sizes_0, x = inputs_13)[name = tensor("op_490")]; - tensor var_492 = sigmoid(x = var_490_1)[name = tensor("op_492")]; - tensor inputs_15 = mul(x = var_490_0, y = var_492)[name = tensor("inputs_15")]; + tensor var_493_split_sizes_0 = const()[name = tensor("op_493_split_sizes_0"), val = tensor([256, 256])]; + tensor var_493_axis_0 = const()[name = tensor("op_493_axis_0"), val = tensor(1)]; + tensor var_493_0, tensor var_493_1 = split(axis = var_493_axis_0, split_sizes = var_493_split_sizes_0, x = inputs_13)[name = tensor("op_493")]; + tensor var_495 = sigmoid(x = var_493_1)[name = tensor("op_495")]; + tensor inputs_15 = mul(x = var_493_0, y = var_495)[name = tensor("inputs_15")]; tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; @@ -544,22 +549,22 @@ program(1.0) tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_523_begin_0 = const()[name = tensor("op_523_begin_0"), val = tensor([0, -1, 0])]; - tensor var_523_end_0 = const()[name = tensor("op_523_end_0"), val = tensor([1, 16, 256])]; - tensor var_523_end_mask_0 = const()[name = tensor("op_523_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_43")]; - tensor var_523 = slice_by_index(begin = var_523_begin_0, end = var_523_end_0, end_mask = var_523_end_mask_0, x = conv_out_3)[name = tensor("op_523")]; - tensor var_526_interleave_0 = const()[name = tensor("op_526_interleave_0"), val = tensor(false)]; - tensor var_526 = concat(axis = var_33, interleave = var_526_interleave_0, values = var_523)[name = tensor("op_526")]; - tensor input_71 = add(x = x_9, y = var_526)[name = tensor("input_71")]; + tensor var_526_begin_0 = const()[name = tensor("op_526_begin_0"), val = tensor([0, -1, 0])]; + tensor var_526_end_0 = const()[name = tensor("op_526_end_0"), val = tensor([1, 16, 256])]; + tensor var_526_end_mask_0 = const()[name = tensor("op_526_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_526 = slice_by_index(begin = var_526_begin_0, end = var_526_end_0, end_mask = var_526_end_mask_0, x = conv_out_3)[name = tensor("op_526")]; + tensor var_528_perm_0 = const()[name = tensor("op_528_perm_0"), val = tensor([1, 0, 2])]; + tensor var_528 = transpose(perm = var_528_perm_0, x = var_526)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_528)[name = tensor("input_71")]; tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; - tensor var_549 = const()[name = tensor("op_549"), val = tensor(0x1p-1)]; - tensor var_550 = mul(x = input_79, y = var_549)[name = tensor("op_550")]; - tensor input_81 = add(x = var_550, y = input_71)[name = tensor("input_81")]; + tensor var_551 = const()[name = tensor("op_551"), val = tensor(0x1p-1)]; + tensor var_552 = mul(x = input_79, y = var_551)[name = tensor("op_552")]; + tensor input_81 = add(x = var_552, y = input_71)[name = tensor("input_81")]; tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; @@ -567,9 +572,9 @@ program(1.0) tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; - tensor var_579 = const()[name = tensor("op_579"), val = tensor(0x1p-1)]; - tensor var_580 = mul(x = input_91, y = var_579)[name = tensor("op_580")]; - tensor input_93 = add(x = var_580, y = input_83)[name = tensor("input_93")]; + tensor var_581 = const()[name = tensor("op_581"), val = tensor(0x1p-1)]; + tensor var_582 = mul(x = input_91, y = var_581)[name = tensor("op_582")]; + tensor input_93 = add(x = var_582, y = input_83)[name = tensor("input_93")]; tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; @@ -582,19 +587,19 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_594 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; - tensor var_595 = const()[name = tensor("op_595"), val = tensor([1, 1, 4, 64])]; - tensor var_596 = reshape(shape = var_595, x = var_594)[name = tensor("op_596")]; + tensor var_596 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_597 = const()[name = tensor("op_597"), val = tensor([1, 1, 4, 64])]; + tensor var_598 = reshape(shape = var_597, x = var_596)[name = tensor("op_598")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_600 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; - tensor var_601 = const()[name = tensor("op_601"), val = tensor(0x1p-3)]; - tensor var_602 = mul(x = var_600, y = var_601)[name = tensor("op_602")]; - tensor var_603 = const()[name = tensor("op_603"), val = tensor([1, 1, 4, 64])]; - tensor var_604 = reshape(shape = var_603, x = var_602)[name = tensor("op_604")]; + tensor var_602 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_603 = const()[name = tensor("op_603"), val = tensor(0x1p-3)]; + tensor var_604 = mul(x = var_602, y = var_603)[name = tensor("op_604")]; + tensor var_605 = const()[name = tensor("op_605"), val = tensor([1, 1, 4, 64])]; + tensor var_606 = reshape(shape = var_605, x = var_604)[name = tensor("op_606")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_608 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; - tensor var_609 = const()[name = tensor("op_609"), val = tensor([1, 1, 4, 64])]; - tensor var_610 = reshape(shape = var_609, x = var_608)[name = tensor("op_610")]; + tensor var_610 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_611 = const()[name = tensor("op_611"), val = tensor([1, 1, 4, 64])]; + tensor var_612 = reshape(shape = var_611, x = var_610)[name = tensor("op_612")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; @@ -602,84 +607,86 @@ program(1.0) tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_604)[name = tensor("transpose_41")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_596)[name = tensor("transpose_42")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_606)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_598)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_620 = const()[name = tensor("op_620"), val = tensor([1, 1])]; - tensor var_621 = reshape(shape = var_620, x = sqrt_s_t_5)[name = tensor("op_621")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_621)[name = tensor("M_5")]; - tensor var_623 = mul(x = qk_5, y = M_5)[name = tensor("op_623")]; + tensor var_622 = const()[name = tensor("op_622"), val = tensor([1, 1])]; + tensor var_623 = reshape(shape = var_622, x = sqrt_s_t_5)[name = tensor("op_623")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_623)[name = tensor("M_5")]; + tensor var_625 = mul(x = qk_5, y = M_5)[name = tensor("op_625")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_610)[name = tensor("transpose_40")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_623, y = v_5)[name = tensor("inner_5")]; - tensor var_625_transpose_x_0 = const()[name = tensor("op_625_transpose_x_0"), val = tensor(false)]; - tensor var_625_transpose_y_0 = const()[name = tensor("op_625_transpose_y_0"), val = tensor(false)]; - tensor var_625 = matmul(transpose_x = var_625_transpose_x_0, transpose_y = var_625_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_625")]; - tensor var_626 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_626")]; - tensor var_627 = const()[name = tensor("op_627"), val = tensor([1, 1, 1, 1])]; - tensor var_628 = reshape(shape = var_627, x = var_626)[name = tensor("op_628")]; - tensor cross_5 = mul(x = var_625, y = var_628)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_612)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_625, y = v_5)[name = tensor("inner_5")]; + tensor var_627_transpose_x_0 = const()[name = tensor("op_627_transpose_x_0"), val = tensor(false)]; + tensor var_627_transpose_y_0 = const()[name = tensor("op_627_transpose_y_0"), val = tensor(false)]; + tensor var_627 = matmul(transpose_x = var_627_transpose_x_0, transpose_y = var_627_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_627")]; + tensor var_628 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_628")]; + tensor var_629 = const()[name = tensor("op_629"), val = tensor([1, 1, 1, 1])]; + tensor var_630 = reshape(shape = var_629, x = var_628)[name = tensor("op_630")]; + tensor cross_5 = mul(x = var_627, y = var_630)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_631 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_631")]; - tensor var_633_transpose_x_1 = const()[name = tensor("op_633_transpose_x_1"), val = tensor(true)]; - tensor var_633_transpose_y_1 = const()[name = tensor("op_633_transpose_y_1"), val = tensor(false)]; - tensor var_633 = matmul(transpose_x = var_633_transpose_x_1, transpose_y = var_633_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_633")]; - tensor new_kv_unnorm_5 = add(x = var_631, y = var_633)[name = tensor("new_kv_unnorm_5")]; - tensor var_635 = const()[name = tensor("op_635"), val = tensor(0x1p+0)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_635)[name = tensor("new_scale_5")]; - tensor var_637 = sqrt(x = new_scale_5)[name = tensor("op_637")]; - tensor var_638 = real_div(x = new_kv_unnorm_5, y = var_637)[name = tensor("op_638")]; - tensor var_639_perm_0 = const()[name = tensor("op_639_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_633 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_633")]; + tensor var_635_transpose_x_1 = const()[name = tensor("op_635_transpose_x_1"), val = tensor(true)]; + tensor var_635_transpose_y_1 = const()[name = tensor("op_635_transpose_y_1"), val = tensor(false)]; + tensor var_635 = matmul(transpose_x = var_635_transpose_x_1, transpose_y = var_635_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_635")]; + tensor new_kv_unnorm_5 = add(x = var_633, y = var_635)[name = tensor("new_kv_unnorm_5")]; + tensor var_637 = const()[name = tensor("op_637"), val = tensor(0x1p+0)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_637)[name = tensor("new_scale_5")]; + tensor var_639 = sqrt(x = new_scale_5)[name = tensor("op_639")]; + tensor var_640 = real_div(x = new_kv_unnorm_5, y = var_639)[name = tensor("op_640")]; + tensor var_641_perm_0 = const()[name = tensor("op_641_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_639 = transpose(perm = var_639_perm_0, x = out_13)[name = tensor("transpose_39")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_639)[name = tensor("out_15")]; - tensor var_643 = const()[name = tensor("op_643"), val = tensor([1, 1, 256])]; - tensor out_17 = reshape(shape = var_643, x = out_15)[name = tensor("out_17")]; - tensor var_645 = silu(x = input_97)[name = tensor("op_645")]; - tensor input_99 = mul(x = var_645, y = out_17)[name = tensor("input_99")]; + tensor var_641 = transpose(perm = var_641_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_641)[name = tensor("out_15")]; + tensor var_645 = const()[name = tensor("op_645"), val = tensor([1, 1, 256])]; + tensor out_17 = reshape(shape = var_645, x = out_15)[name = tensor("out_17")]; + tensor var_647 = silu(x = input_97)[name = tensor("op_647")]; + tensor input_99 = mul(x = var_647, y = out_17)[name = tensor("input_99")]; tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor window_9_begin_0 = const()[name = tensor("window_9_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_9_end_0 = const()[name = tensor("window_9_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_9_end_mask_0 = const()[name = tensor("window_9_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_9_squeeze_mask_0 = const()[name = tensor("window_9_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_9 = slice_by_index(begin = window_9_begin_0, end = window_9_end_0, end_mask = window_9_end_mask_0, squeeze_mask = window_9_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_9")]; tensor filled_5_begin_0 = const()[name = tensor("filled_5_begin_0"), val = tensor([2, 0])]; tensor filled_5_end_0 = const()[name = tensor("filled_5_end_0"), val = tensor([3, 1])]; tensor filled_5_end_mask_0 = const()[name = tensor("filled_5_end_mask_0"), val = tensor([false, true])]; tensor filled_5_squeeze_mask_0 = const()[name = tensor("filled_5_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_5 = slice_by_index(begin = filled_5_begin_0, end = filled_5_end_0, end_mask = filled_5_end_mask_0, squeeze_mask = filled_5_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_5")]; - tensor var_657_begin_0 = const()[name = tensor("op_657_begin_0"), val = tensor([0, 1, 0])]; - tensor var_657_end_0 = const()[name = tensor("op_657_end_0"), val = tensor([1, 16, 256])]; - tensor var_657_end_mask_0 = const()[name = tensor("op_657_end_mask_0"), val = tensor([true, true, true])]; - tensor var_657 = slice_by_index(begin = var_657_begin_0, end = var_657_end_0, end_mask = var_657_end_mask_0, x = window_5)[name = tensor("op_657")]; + tensor var_659_begin_0 = const()[name = tensor("op_659_begin_0"), val = tensor([0, 1, 0])]; + tensor var_659_end_0 = const()[name = tensor("op_659_end_0"), val = tensor([1, 16, 256])]; + tensor var_659_end_mask_0 = const()[name = tensor("op_659_end_mask_0"), val = tensor([true, true, true])]; + tensor var_659 = slice_by_index(begin = var_659_begin_0, end = var_659_end_0, end_mask = var_659_end_mask_0, x = window_9)[name = tensor("op_659")]; tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_657, x_15))[name = tensor("rolled_5")]; + tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_659, x_15))[name = tensor("rolled_5")]; tensor clip_4 = clip(alpha = var_21, beta = var_20, x = filled_5)[name = tensor("clip_4")]; - tensor var_662 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_662")]; - tensor var_663_dtype_0 = const()[name = tensor("op_663_dtype_0"), val = tensor("fp32")]; - tensor var_664 = const()[name = tensor("op_664"), val = tensor([1, 16, 1])]; - tensor var_663 = cast(dtype = var_663_dtype_0, x = var_662)[name = tensor("cast_123")]; - tensor fill_mask_5 = reshape(shape = var_664, x = var_663)[name = tensor("fill_mask_5")]; + tensor var_664 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_664")]; + tensor var_665_dtype_0 = const()[name = tensor("op_665_dtype_0"), val = tensor("fp32")]; + tensor var_666 = const()[name = tensor("op_666"), val = tensor([1, 16, 1])]; + tensor var_665 = cast(dtype = var_665_dtype_0, x = var_664)[name = tensor("cast_123")]; + tensor fill_mask_5 = reshape(shape = var_666, x = var_665)[name = tensor("fill_mask_5")]; tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = x_15)[name = tensor("x_expanded_5")]; - tensor var_668 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_668")]; - tensor var_669 = mul(x = window_5, y = var_668)[name = tensor("op_669")]; - tensor var_670 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_670")]; - tensor filled_window_5 = add(x = var_669, y = var_670)[name = tensor("filled_window_5")]; - tensor var_672 = greater_equal(x = filled_5, y = var_16)[name = tensor("op_672")]; - tensor var_673_dtype_0 = const()[name = tensor("op_673_dtype_0"), val = tensor("fp32")]; - tensor var_674 = const()[name = tensor("op_674"), val = tensor([1, 1, 1])]; - tensor var_673 = cast(dtype = var_673_dtype_0, x = var_672)[name = tensor("cast_122")]; - tensor use_roll_5 = reshape(shape = var_674, x = var_673)[name = tensor("use_roll_5")]; - tensor var_676 = sub(x = var_17, y = use_roll_5)[name = tensor("op_676")]; - tensor var_677 = mul(x = filled_window_5, y = var_676)[name = tensor("op_677")]; - tensor var_678 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_678")]; - tensor input_101 = add(x = var_677, y = var_678)[name = tensor("input_101")]; - tensor var_680 = const()[name = tensor("op_680"), val = tensor(0x1p+0)]; - tensor var_681 = add(x = filled_5, y = var_680)[name = tensor("op_681")]; - tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_681)[name = tensor("clip_5")]; + tensor var_670 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_670")]; + tensor var_671 = mul(x = window_9, y = var_670)[name = tensor("op_671")]; + tensor var_672 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_672")]; + tensor filled_window_5 = add(x = var_671, y = var_672)[name = tensor("filled_window_5")]; + tensor var_674 = greater_equal(x = filled_5, y = var_16)[name = tensor("op_674")]; + tensor var_675_dtype_0 = const()[name = tensor("op_675_dtype_0"), val = tensor("fp32")]; + tensor var_676 = const()[name = tensor("op_676"), val = tensor([1, 1, 1])]; + tensor var_675 = cast(dtype = var_675_dtype_0, x = var_674)[name = tensor("cast_122")]; + tensor use_roll_5 = reshape(shape = var_676, x = var_675)[name = tensor("use_roll_5")]; + tensor var_678 = sub(x = var_17, y = use_roll_5)[name = tensor("op_678")]; + tensor var_679 = mul(x = filled_window_5, y = var_678)[name = tensor("op_679")]; + tensor var_680 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_680")]; + tensor window_11 = add(x = var_679, y = var_680)[name = tensor("window_11")]; + tensor var_682 = const()[name = tensor("op_682"), val = tensor(0x1p+0)]; + tensor var_683 = add(x = filled_5, y = var_682)[name = tensor("op_683")]; + tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_683)[name = tensor("clip_5")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_30, interleave = input_101_interleave_0, values = window_11)[name = tensor("input_101")]; tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; @@ -688,13 +695,13 @@ program(1.0) tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; - tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_38")]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; - tensor var_702_split_sizes_0 = const()[name = tensor("op_702_split_sizes_0"), val = tensor([256, 256])]; - tensor var_702_axis_0 = const()[name = tensor("op_702_axis_0"), val = tensor(1)]; - tensor var_702_0, tensor var_702_1 = split(axis = var_702_axis_0, split_sizes = var_702_split_sizes_0, x = inputs_23)[name = tensor("op_702")]; - tensor var_704 = sigmoid(x = var_702_1)[name = tensor("op_704")]; - tensor inputs_25 = mul(x = var_702_0, y = var_704)[name = tensor("inputs_25")]; + tensor var_706_split_sizes_0 = const()[name = tensor("op_706_split_sizes_0"), val = tensor([256, 256])]; + tensor var_706_axis_0 = const()[name = tensor("op_706_axis_0"), val = tensor(1)]; + tensor var_706_0, tensor var_706_1 = split(axis = var_706_axis_0, split_sizes = var_706_split_sizes_0, x = inputs_23)[name = tensor("op_706")]; + tensor var_708 = sigmoid(x = var_706_1)[name = tensor("op_708")]; + tensor inputs_25 = mul(x = var_706_0, y = var_708)[name = tensor("inputs_25")]; tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; @@ -714,22 +721,22 @@ program(1.0) tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_735_begin_0 = const()[name = tensor("op_735_begin_0"), val = tensor([0, -1, 0])]; - tensor var_735_end_0 = const()[name = tensor("op_735_end_0"), val = tensor([1, 16, 256])]; - tensor var_735_end_mask_0 = const()[name = tensor("op_735_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_37")]; - tensor var_735 = slice_by_index(begin = var_735_begin_0, end = var_735_end_0, end_mask = var_735_end_mask_0, x = conv_out_5)[name = tensor("op_735")]; - tensor var_738_interleave_0 = const()[name = tensor("op_738_interleave_0"), val = tensor(false)]; - tensor var_738 = concat(axis = var_33, interleave = var_738_interleave_0, values = var_735)[name = tensor("op_738")]; - tensor input_111 = add(x = x_15, y = var_738)[name = tensor("input_111")]; + tensor var_739_begin_0 = const()[name = tensor("op_739_begin_0"), val = tensor([0, -1, 0])]; + tensor var_739_end_0 = const()[name = tensor("op_739_end_0"), val = tensor([1, 16, 256])]; + tensor var_739_end_mask_0 = const()[name = tensor("op_739_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_739 = slice_by_index(begin = var_739_begin_0, end = var_739_end_0, end_mask = var_739_end_mask_0, x = conv_out_5)[name = tensor("op_739")]; + tensor var_741_perm_0 = const()[name = tensor("op_741_perm_0"), val = tensor([1, 0, 2])]; + tensor var_741 = transpose(perm = var_741_perm_0, x = var_739)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_741)[name = tensor("input_111")]; tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; - tensor var_761 = const()[name = tensor("op_761"), val = tensor(0x1p-1)]; - tensor var_762 = mul(x = input_119, y = var_761)[name = tensor("op_762")]; - tensor input_121 = add(x = var_762, y = input_111)[name = tensor("input_121")]; + tensor var_764 = const()[name = tensor("op_764"), val = tensor(0x1p-1)]; + tensor var_765 = mul(x = input_119, y = var_764)[name = tensor("op_765")]; + tensor input_121 = add(x = var_765, y = input_111)[name = tensor("input_121")]; tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; @@ -737,9 +744,9 @@ program(1.0) tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; - tensor var_791 = const()[name = tensor("op_791"), val = tensor(0x1p-1)]; - tensor var_792 = mul(x = input_131, y = var_791)[name = tensor("op_792")]; - tensor input_133 = add(x = var_792, y = input_123)[name = tensor("input_133")]; + tensor var_794 = const()[name = tensor("op_794"), val = tensor(0x1p-1)]; + tensor var_795 = mul(x = input_131, y = var_794)[name = tensor("op_795")]; + tensor input_133 = add(x = var_795, y = input_123)[name = tensor("input_133")]; tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; @@ -752,19 +759,19 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_806 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; - tensor var_807 = const()[name = tensor("op_807"), val = tensor([1, 1, 4, 64])]; - tensor var_808 = reshape(shape = var_807, x = var_806)[name = tensor("op_808")]; + tensor var_809 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_810 = const()[name = tensor("op_810"), val = tensor([1, 1, 4, 64])]; + tensor var_811 = reshape(shape = var_810, x = var_809)[name = tensor("op_811")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_812 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; - tensor var_813 = const()[name = tensor("op_813"), val = tensor(0x1p-3)]; - tensor var_814 = mul(x = var_812, y = var_813)[name = tensor("op_814")]; - tensor var_815 = const()[name = tensor("op_815"), val = tensor([1, 1, 4, 64])]; - tensor var_816 = reshape(shape = var_815, x = var_814)[name = tensor("op_816")]; + tensor var_815 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_816 = const()[name = tensor("op_816"), val = tensor(0x1p-3)]; + tensor var_817 = mul(x = var_815, y = var_816)[name = tensor("op_817")]; + tensor var_818 = const()[name = tensor("op_818"), val = tensor([1, 1, 4, 64])]; + tensor var_819 = reshape(shape = var_818, x = var_817)[name = tensor("op_819")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_820 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; - tensor var_821 = const()[name = tensor("op_821"), val = tensor([1, 1, 4, 64])]; - tensor var_822 = reshape(shape = var_821, x = var_820)[name = tensor("op_822")]; + tensor var_823 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_824 = const()[name = tensor("op_824"), val = tensor([1, 1, 4, 64])]; + tensor var_825 = reshape(shape = var_824, x = var_823)[name = tensor("op_825")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; @@ -772,84 +779,86 @@ program(1.0) tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_816)[name = tensor("transpose_35")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_808)[name = tensor("transpose_36")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_819)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_811)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_832 = const()[name = tensor("op_832"), val = tensor([1, 1])]; - tensor var_833 = reshape(shape = var_832, x = sqrt_s_t_7)[name = tensor("op_833")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_833)[name = tensor("M_7")]; - tensor var_835 = mul(x = qk_7, y = M_7)[name = tensor("op_835")]; + tensor var_835 = const()[name = tensor("op_835"), val = tensor([1, 1])]; + tensor var_836 = reshape(shape = var_835, x = sqrt_s_t_7)[name = tensor("op_836")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_836)[name = tensor("M_7")]; + tensor var_838 = mul(x = qk_7, y = M_7)[name = tensor("op_838")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_822)[name = tensor("transpose_34")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_835, y = v_7)[name = tensor("inner_7")]; - tensor var_837_transpose_x_0 = const()[name = tensor("op_837_transpose_x_0"), val = tensor(false)]; - tensor var_837_transpose_y_0 = const()[name = tensor("op_837_transpose_y_0"), val = tensor(false)]; - tensor var_837 = matmul(transpose_x = var_837_transpose_x_0, transpose_y = var_837_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_837")]; - tensor var_838 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_838")]; - tensor var_839 = const()[name = tensor("op_839"), val = tensor([1, 1, 1, 1])]; - tensor var_840 = reshape(shape = var_839, x = var_838)[name = tensor("op_840")]; - tensor cross_7 = mul(x = var_837, y = var_840)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_825)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_838, y = v_7)[name = tensor("inner_7")]; + tensor var_840_transpose_x_0 = const()[name = tensor("op_840_transpose_x_0"), val = tensor(false)]; + tensor var_840_transpose_y_0 = const()[name = tensor("op_840_transpose_y_0"), val = tensor(false)]; + tensor var_840 = matmul(transpose_x = var_840_transpose_x_0, transpose_y = var_840_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_840")]; + tensor var_841 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_841")]; + tensor var_842 = const()[name = tensor("op_842"), val = tensor([1, 1, 1, 1])]; + tensor var_843 = reshape(shape = var_842, x = var_841)[name = tensor("op_843")]; + tensor cross_7 = mul(x = var_840, y = var_843)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_843 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_843")]; - tensor var_845_transpose_x_1 = const()[name = tensor("op_845_transpose_x_1"), val = tensor(true)]; - tensor var_845_transpose_y_1 = const()[name = tensor("op_845_transpose_y_1"), val = tensor(false)]; - tensor var_845 = matmul(transpose_x = var_845_transpose_x_1, transpose_y = var_845_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_845")]; - tensor new_kv_unnorm_7 = add(x = var_843, y = var_845)[name = tensor("new_kv_unnorm_7")]; - tensor var_847 = const()[name = tensor("op_847"), val = tensor(0x1p+0)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_847)[name = tensor("new_scale_7")]; - tensor var_849 = sqrt(x = new_scale_7)[name = tensor("op_849")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_849)[name = tensor("nkv_1")]; - tensor var_851_perm_0 = const()[name = tensor("op_851_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_846 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_846")]; + tensor var_848_transpose_x_1 = const()[name = tensor("op_848_transpose_x_1"), val = tensor(true)]; + tensor var_848_transpose_y_1 = const()[name = tensor("op_848_transpose_y_1"), val = tensor(false)]; + tensor var_848 = matmul(transpose_x = var_848_transpose_x_1, transpose_y = var_848_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_848")]; + tensor new_kv_unnorm_7 = add(x = var_846, y = var_848)[name = tensor("new_kv_unnorm_7")]; + tensor var_850 = const()[name = tensor("op_850"), val = tensor(0x1p+0)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_850)[name = tensor("new_scale_7")]; + tensor var_852 = sqrt(x = new_scale_7)[name = tensor("op_852")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_852)[name = tensor("nkv_1")]; + tensor var_854_perm_0 = const()[name = tensor("op_854_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_851 = transpose(perm = var_851_perm_0, x = out_19)[name = tensor("transpose_33")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_851)[name = tensor("out_21")]; - tensor var_855 = const()[name = tensor("op_855"), val = tensor([1, 1, 256])]; - tensor out_23 = reshape(shape = var_855, x = out_21)[name = tensor("out_23")]; - tensor var_857 = silu(x = input_137)[name = tensor("op_857")]; - tensor input_139 = mul(x = var_857, y = out_23)[name = tensor("input_139")]; + tensor var_854 = transpose(perm = var_854_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_854)[name = tensor("out_21")]; + tensor var_858 = const()[name = tensor("op_858"), val = tensor([1, 1, 256])]; + tensor out_23 = reshape(shape = var_858, x = out_21)[name = tensor("out_23")]; + tensor var_860 = silu(x = input_137)[name = tensor("op_860")]; + tensor input_139 = mul(x = var_860, y = out_23)[name = tensor("input_139")]; tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor window_13_begin_0 = const()[name = tensor("window_13_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_13_end_0 = const()[name = tensor("window_13_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_13_end_mask_0 = const()[name = tensor("window_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_13_squeeze_mask_0 = const()[name = tensor("window_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_13 = slice_by_index(begin = window_13_begin_0, end = window_13_end_0, end_mask = window_13_end_mask_0, squeeze_mask = window_13_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_13")]; tensor filled_7_begin_0 = const()[name = tensor("filled_7_begin_0"), val = tensor([3, 0])]; tensor filled_7_end_0 = const()[name = tensor("filled_7_end_0"), val = tensor([4, 1])]; tensor filled_7_end_mask_0 = const()[name = tensor("filled_7_end_mask_0"), val = tensor([false, true])]; tensor filled_7_squeeze_mask_0 = const()[name = tensor("filled_7_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_7 = slice_by_index(begin = filled_7_begin_0, end = filled_7_end_0, end_mask = filled_7_end_mask_0, squeeze_mask = filled_7_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_7")]; - tensor var_869_begin_0 = const()[name = tensor("op_869_begin_0"), val = tensor([0, 1, 0])]; - tensor var_869_end_0 = const()[name = tensor("op_869_end_0"), val = tensor([1, 16, 256])]; - tensor var_869_end_mask_0 = const()[name = tensor("op_869_end_mask_0"), val = tensor([true, true, true])]; - tensor var_869 = slice_by_index(begin = var_869_begin_0, end = var_869_end_0, end_mask = var_869_end_mask_0, x = window)[name = tensor("op_869")]; + tensor var_872_begin_0 = const()[name = tensor("op_872_begin_0"), val = tensor([0, 1, 0])]; + tensor var_872_end_0 = const()[name = tensor("op_872_end_0"), val = tensor([1, 16, 256])]; + tensor var_872_end_mask_0 = const()[name = tensor("op_872_end_mask_0"), val = tensor([true, true, true])]; + tensor var_872 = slice_by_index(begin = var_872_begin_0, end = var_872_end_0, end_mask = var_872_end_mask_0, x = window_13)[name = tensor("op_872")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_869, x_21))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_872, x_21))[name = tensor("rolled")]; tensor clip_6 = clip(alpha = var_21, beta = var_20, x = filled_7)[name = tensor("clip_6")]; - tensor var_874 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_874")]; - tensor var_875_dtype_0 = const()[name = tensor("op_875_dtype_0"), val = tensor("fp32")]; - tensor var_876 = const()[name = tensor("op_876"), val = tensor([1, 16, 1])]; - tensor var_875 = cast(dtype = var_875_dtype_0, x = var_874)[name = tensor("cast_121")]; - tensor fill_mask = reshape(shape = var_876, x = var_875)[name = tensor("fill_mask")]; + tensor var_877 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_877")]; + tensor var_878_dtype_0 = const()[name = tensor("op_878_dtype_0"), val = tensor("fp32")]; + tensor var_879 = const()[name = tensor("op_879"), val = tensor([1, 16, 1])]; + tensor var_878 = cast(dtype = var_878_dtype_0, x = var_877)[name = tensor("cast_121")]; + tensor fill_mask = reshape(shape = var_879, x = var_878)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; tensor x_expanded = tile(reps = x_expanded_reps_0, x = x_21)[name = tensor("x_expanded")]; - tensor var_880 = sub(x = var_17, y = fill_mask)[name = tensor("op_880")]; - tensor var_881 = mul(x = window, y = var_880)[name = tensor("op_881")]; - tensor var_882 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_882")]; - tensor filled_window = add(x = var_881, y = var_882)[name = tensor("filled_window")]; - tensor var_884 = greater_equal(x = filled_7, y = var_16)[name = tensor("op_884")]; - tensor var_885_dtype_0 = const()[name = tensor("op_885_dtype_0"), val = tensor("fp32")]; - tensor var_886 = const()[name = tensor("op_886"), val = tensor([1, 1, 1])]; - tensor var_885 = cast(dtype = var_885_dtype_0, x = var_884)[name = tensor("cast_120")]; - tensor use_roll = reshape(shape = var_886, x = var_885)[name = tensor("use_roll")]; - tensor var_888 = sub(x = var_17, y = use_roll)[name = tensor("op_888")]; - tensor var_889 = mul(x = filled_window, y = var_888)[name = tensor("op_889")]; - tensor var_890 = mul(x = rolled, y = use_roll)[name = tensor("op_890")]; - tensor input_141 = add(x = var_889, y = var_890)[name = tensor("input_141")]; - tensor var_892 = const()[name = tensor("op_892"), val = tensor(0x1p+0)]; - tensor var_893 = add(x = filled_7, y = var_892)[name = tensor("op_893")]; - tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_893)[name = tensor("clip_7")]; + tensor var_883 = sub(x = var_17, y = fill_mask)[name = tensor("op_883")]; + tensor var_884 = mul(x = window_13, y = var_883)[name = tensor("op_884")]; + tensor var_885 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_885")]; + tensor filled_window = add(x = var_884, y = var_885)[name = tensor("filled_window")]; + tensor var_887 = greater_equal(x = filled_7, y = var_16)[name = tensor("op_887")]; + tensor var_888_dtype_0 = const()[name = tensor("op_888_dtype_0"), val = tensor("fp32")]; + tensor var_889 = const()[name = tensor("op_889"), val = tensor([1, 1, 1])]; + tensor var_888 = cast(dtype = var_888_dtype_0, x = var_887)[name = tensor("cast_120")]; + tensor use_roll = reshape(shape = var_889, x = var_888)[name = tensor("use_roll")]; + tensor var_891 = sub(x = var_17, y = use_roll)[name = tensor("op_891")]; + tensor var_892 = mul(x = filled_window, y = var_891)[name = tensor("op_892")]; + tensor var_893 = mul(x = rolled, y = use_roll)[name = tensor("op_893")]; + tensor window = add(x = var_892, y = var_893)[name = tensor("window")]; + tensor var_895 = const()[name = tensor("op_895"), val = tensor(0x1p+0)]; + tensor var_896 = add(x = filled_7, y = var_895)[name = tensor("op_896")]; + tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_896)[name = tensor("clip_7")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_30, interleave = input_141_interleave_0, values = window)[name = tensor("input_141")]; tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; @@ -858,13 +867,13 @@ program(1.0) tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; - tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_32")]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; - tensor var_914_split_sizes_0 = const()[name = tensor("op_914_split_sizes_0"), val = tensor([256, 256])]; - tensor var_914_axis_0 = const()[name = tensor("op_914_axis_0"), val = tensor(1)]; - tensor var_914_0, tensor var_914_1 = split(axis = var_914_axis_0, split_sizes = var_914_split_sizes_0, x = inputs_33)[name = tensor("op_914")]; - tensor var_916 = sigmoid(x = var_914_1)[name = tensor("op_916")]; - tensor inputs_35 = mul(x = var_914_0, y = var_916)[name = tensor("inputs_35")]; + tensor var_919_split_sizes_0 = const()[name = tensor("op_919_split_sizes_0"), val = tensor([256, 256])]; + tensor var_919_axis_0 = const()[name = tensor("op_919_axis_0"), val = tensor(1)]; + tensor var_919_0, tensor var_919_1 = split(axis = var_919_axis_0, split_sizes = var_919_split_sizes_0, x = inputs_33)[name = tensor("op_919")]; + tensor var_921 = sigmoid(x = var_919_1)[name = tensor("op_921")]; + tensor inputs_35 = mul(x = var_919_0, y = var_921)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; @@ -884,22 +893,22 @@ program(1.0) tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_947_begin_0 = const()[name = tensor("op_947_begin_0"), val = tensor([0, -1, 0])]; - tensor var_947_end_0 = const()[name = tensor("op_947_end_0"), val = tensor([1, 16, 256])]; - tensor var_947_end_mask_0 = const()[name = tensor("op_947_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_31")]; - tensor var_947 = slice_by_index(begin = var_947_begin_0, end = var_947_end_0, end_mask = var_947_end_mask_0, x = conv_out_7)[name = tensor("op_947")]; - tensor var_950_interleave_0 = const()[name = tensor("op_950_interleave_0"), val = tensor(false)]; - tensor var_950 = concat(axis = var_33, interleave = var_950_interleave_0, values = var_947)[name = tensor("op_950")]; - tensor input_151 = add(x = x_21, y = var_950)[name = tensor("input_151")]; + tensor var_952_begin_0 = const()[name = tensor("op_952_begin_0"), val = tensor([0, -1, 0])]; + tensor var_952_end_0 = const()[name = tensor("op_952_end_0"), val = tensor([1, 16, 256])]; + tensor var_952_end_mask_0 = const()[name = tensor("op_952_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_952 = slice_by_index(begin = var_952_begin_0, end = var_952_end_0, end_mask = var_952_end_mask_0, x = conv_out_7)[name = tensor("op_952")]; + tensor var_954_perm_0 = const()[name = tensor("op_954_perm_0"), val = tensor([1, 0, 2])]; + tensor var_954 = transpose(perm = var_954_perm_0, x = var_952)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_954)[name = tensor("input_151")]; tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; - tensor var_973 = const()[name = tensor("op_973"), val = tensor(0x1p-1)]; - tensor var_974 = mul(x = input_159, y = var_973)[name = tensor("op_974")]; - tensor input_161 = add(x = var_974, y = input_151)[name = tensor("input_161")]; + tensor var_977 = const()[name = tensor("op_977"), val = tensor(0x1p-1)]; + tensor var_978 = mul(x = input_159, y = var_977)[name = tensor("op_978")]; + tensor input_161 = add(x = var_978, y = input_151)[name = tensor("input_161")]; tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; @@ -912,44 +921,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_992_begin_0 = const()[name = tensor("op_992_begin_0"), val = tensor([0, 0, 1])]; - tensor var_992_end_0 = const()[name = tensor("op_992_end_0"), val = tensor([1, 256, 19])]; - tensor var_992_end_mask_0 = const()[name = tensor("op_992_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_992_begin_0, end = var_992_end_0, end_mask = var_992_end_mask_0, x = cat)[name = tensor("op_992")]; + tensor var_996_begin_0 = const()[name = tensor("op_996_begin_0"), val = tensor([0, 0, 1])]; + tensor var_996_end_0 = const()[name = tensor("op_996_end_0"), val = tensor([1, 256, 19])]; + tensor var_996_end_mask_0 = const()[name = tensor("op_996_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_996_begin_0, end = var_996_end_0, end_mask = var_996_end_mask_0, x = cat)[name = tensor("op_996")]; tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; - tensor var_994 = const()[name = tensor("op_994"), val = tensor([-1])]; + tensor var_998 = const()[name = tensor("op_998"), val = tensor([-1])]; tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_995 = reduce_l2_norm(axes = var_994, keep_dims = var_36, x = input_163)[name = tensor("op_995")]; + tensor var_999 = reduce_l2_norm(axes = var_998, keep_dims = var_36, x = input_163)[name = tensor("op_999")]; tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; - tensor clip_8 = clip(alpha = var_12, beta = const_12, x = var_995)[name = tensor("clip_8")]; + tensor clip_8 = clip(alpha = var_12, beta = const_12, x = var_999)[name = tensor("clip_8")]; tensor emb = real_div(x = input_163, y = clip_8)[name = tensor("emb")]; - tensor var_999_axis_0 = const()[name = tensor("op_999_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_999_axis_0, values = (var_214, var_426, var_638, nkv_1))[name = tensor("op_999")]; - tensor var_1001_axis_0 = const()[name = tensor("op_1001_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_1001_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1001")]; tensor var_1003_axis_0 = const()[name = tensor("op_1003_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_1003_axis_0, values = (input_21, input_61, input_101, input_141))[name = tensor("op_1003")]; + tensor enc_kv_new = stack(axis = var_1003_axis_0, values = (var_214, var_427, var_640, nkv_1))[name = tensor("op_1003")]; tensor var_1005_axis_0 = const()[name = tensor("op_1005_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_1005_axis_0, values = (clip_1, clip_3, clip_5, clip_7))[name = tensor("op_1005")]; - tensor var_1015 = const()[name = tensor("op_1015"), val = tensor(0x1.5798eep-27)]; - tensor var_1021 = const()[name = tensor("op_1021"), val = tensor(0x1.4f8b58p-17)]; - tensor var_1023 = const()[name = tensor("op_1023"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_1024 = const()[name = tensor("op_1024"), val = tensor(true)]; - tensor var_1026 = const()[name = tensor("op_1026"), val = tensor(0x1p+0)]; - tensor var_1030 = const()[name = tensor("op_1030"), val = tensor(-1)]; - tensor var_1035 = const()[name = tensor("op_1035"), val = tensor(0)]; - tensor var_1092 = const()[name = tensor("op_1092"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395456)))]; - tensor var_1097_axes_0 = const()[name = tensor("op_1097_axes_0"), val = tensor([2])]; - tensor var_1097 = expand_dims(axes = var_1097_axes_0, x = emb)[name = tensor("op_1097")]; - tensor var_1098 = const()[name = tensor("op_1098"), val = tensor([1, 1, 12, 1])]; - tensor emb_exp = tile(reps = var_1098, x = var_1097)[name = tensor("emb_exp")]; + tensor enc_scale_new = stack(axis = var_1005_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1005")]; + tensor var_1007_axis_0 = const()[name = tensor("op_1007_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1007_axis_0, values = (window_3, window_7, window_11, window))[name = tensor("op_1007")]; + tensor var_1009_axis_0 = const()[name = tensor("op_1009_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1009_axis_0, values = (clip_1, clip_3, clip_5, clip_7))[name = tensor("op_1009")]; + tensor var_1019 = const()[name = tensor("op_1019"), val = tensor(0x1.5798eep-27)]; + tensor var_1024 = const()[name = tensor("op_1024"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1026 = const()[name = tensor("op_1026"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1027 = const()[name = tensor("op_1027"), val = tensor(true)]; + tensor var_1029 = const()[name = tensor("op_1029"), val = tensor(0x1p+0)]; + tensor var_1033 = const()[name = tensor("op_1033"), val = tensor(-1)]; + tensor var_1039 = const()[name = tensor("op_1039"), val = tensor(0)]; + tensor var_1096 = const()[name = tensor("op_1096"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395456)))]; + tensor var_1101_axes_0 = const()[name = tensor("op_1101_axes_0"), val = tensor([2])]; + tensor var_1101 = expand_dims(axes = var_1101_axes_0, x = emb)[name = tensor("op_1101")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 12, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1101)[name = tensor("emb_exp")]; tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; - tensor input_165 = concat(axis = var_1030, interleave = input_165_interleave_0, values = (emb_exp, var_1092))[name = tensor("input_165")]; + tensor input_165 = concat(axis = var_1033, interleave = input_165_interleave_0, values = (emb_exp, var_1096))[name = tensor("input_165")]; tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; - tensor var_1105_perm_0 = const()[name = tensor("op_1105_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1109 = const()[name = tensor("op_1109"), val = tensor([12, 1, 256])]; - tensor var_1105 = transpose(perm = var_1105_perm_0, x = x_27)[name = tensor("transpose_28")]; - tensor x_29 = reshape(shape = var_1109, x = var_1105)[name = tensor("x_29")]; + tensor var_1109_perm_0 = const()[name = tensor("op_1109_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1113 = const()[name = tensor("op_1113"), val = tensor([12, 1, 256])]; + tensor var_1109 = transpose(perm = var_1109_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1113, x = var_1109)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 12, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -960,131 +969,131 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_1117 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; - tensor var_1118 = const()[name = tensor("op_1118"), val = tensor([12, 1, 4, 64])]; - tensor var_1119 = reshape(shape = var_1118, x = var_1117)[name = tensor("op_1119")]; + tensor var_1121 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1122 = const()[name = tensor("op_1122"), val = tensor([12, 1, 4, 64])]; + tensor var_1123 = reshape(shape = var_1122, x = var_1121)[name = tensor("op_1123")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1123 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; - tensor var_1124 = const()[name = tensor("op_1124"), val = tensor(0x1p-3)]; - tensor var_1125 = mul(x = var_1123, y = var_1124)[name = tensor("op_1125")]; - tensor var_1126 = const()[name = tensor("op_1126"), val = tensor([12, 1, 4, 64])]; - tensor var_1127 = reshape(shape = var_1126, x = var_1125)[name = tensor("op_1127")]; + tensor var_1127 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1128 = const()[name = tensor("op_1128"), val = tensor(0x1p-3)]; + tensor var_1129 = mul(x = var_1127, y = var_1128)[name = tensor("op_1129")]; + tensor var_1130 = const()[name = tensor("op_1130"), val = tensor([12, 1, 4, 64])]; + tensor var_1131 = reshape(shape = var_1130, x = var_1129)[name = tensor("op_1131")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1131 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; - tensor var_1132 = const()[name = tensor("op_1132"), val = tensor([12, 1, 4, 64])]; - tensor var_1133 = reshape(shape = var_1132, x = var_1131)[name = tensor("op_1133")]; + tensor var_1135 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1136 = const()[name = tensor("op_1136"), val = tensor([12, 1, 4, 64])]; + tensor var_1137 = reshape(shape = var_1136, x = var_1135)[name = tensor("op_1137")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_1035, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1039, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; - tensor clip_9 = clip(alpha = var_1026, beta = const_20, x = s_t_9)[name = tensor("clip_9")]; + tensor clip_9 = clip(alpha = var_1029, beta = const_20, x = s_t_9)[name = tensor("clip_9")]; tensor sqrt_s_t_9 = sqrt(x = clip_9)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_1127)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_1119)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1131)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1123)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_1145 = const()[name = tensor("op_1145"), val = tensor([1, 1])]; - tensor var_1146 = reshape(shape = var_1145, x = valid_mask)[name = tensor("op_1146")]; - tensor var_1148 = const()[name = tensor("op_1148"), val = tensor([1, 1])]; - tensor var_1149 = reshape(shape = var_1148, x = sqrt_s_t_9)[name = tensor("op_1149")]; - tensor M_9 = real_div(x = var_1146, y = var_1149)[name = tensor("M_9")]; - tensor var_1151 = mul(x = qk_9, y = M_9)[name = tensor("op_1151")]; + tensor var_1149 = const()[name = tensor("op_1149"), val = tensor([1, 1])]; + tensor var_1150 = reshape(shape = var_1149, x = valid_mask)[name = tensor("op_1150")]; + tensor var_1152 = const()[name = tensor("op_1152"), val = tensor([1, 1])]; + tensor var_1153 = reshape(shape = var_1152, x = sqrt_s_t_9)[name = tensor("op_1153")]; + tensor M_9 = real_div(x = var_1150, y = var_1153)[name = tensor("M_9")]; + tensor var_1155 = mul(x = qk_9, y = M_9)[name = tensor("op_1155")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_1133)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1151, y = v_9)[name = tensor("inner_9")]; - tensor var_1153_transpose_x_0 = const()[name = tensor("op_1153_transpose_x_0"), val = tensor(false)]; - tensor var_1153_transpose_y_0 = const()[name = tensor("op_1153_transpose_y_0"), val = tensor(false)]; - tensor var_1153 = matmul(transpose_x = var_1153_transpose_x_0, transpose_y = var_1153_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1153")]; - tensor var_1154 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1154")]; - tensor var_1155 = const()[name = tensor("op_1155"), val = tensor([1, 1, 1, 1])]; - tensor var_1156 = reshape(shape = var_1155, x = var_1154)[name = tensor("op_1156")]; - tensor cross_9 = mul(x = var_1153, y = var_1156)[name = tensor("cross_9")]; - tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1137)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1155, y = v_9)[name = tensor("inner_9")]; + tensor var_1157_transpose_x_0 = const()[name = tensor("op_1157_transpose_x_0"), val = tensor(false)]; + tensor var_1157_transpose_y_0 = const()[name = tensor("op_1157_transpose_y_0"), val = tensor(false)]; + tensor var_1157 = matmul(transpose_x = var_1157_transpose_x_0, transpose_y = var_1157_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1157")]; + tensor var_1158 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1158")]; tensor var_1159 = const()[name = tensor("op_1159"), val = tensor([1, 1, 1, 1])]; - tensor var_1160 = reshape(shape = var_1159, x = valid_mask)[name = tensor("op_1160")]; - tensor v_masked_1 = mul(x = v_9, y = var_1160)[name = tensor("v_masked_1")]; - tensor var_1162 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1162")]; - tensor var_1164_transpose_x_1 = const()[name = tensor("op_1164_transpose_x_1"), val = tensor(true)]; - tensor var_1164_transpose_y_1 = const()[name = tensor("op_1164_transpose_y_1"), val = tensor(false)]; - tensor var_1164 = matmul(transpose_x = var_1164_transpose_x_1, transpose_y = var_1164_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1164")]; - tensor new_kv_unnorm_9 = add(x = var_1162, y = var_1164)[name = tensor("new_kv_unnorm_9")]; - tensor var_1166_keep_dims_0 = const()[name = tensor("op_1166_keep_dims_0"), val = tensor(false)]; - tensor var_1166 = reduce_sum(keep_dims = var_1166_keep_dims_0, x = valid_mask)[name = tensor("op_1166")]; - tensor var_1167 = const()[name = tensor("op_1167"), val = tensor([1])]; - tensor var_1168 = reshape(shape = var_1167, x = var_1166)[name = tensor("op_1168")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_1168)[name = tensor("new_scale_9")]; + tensor var_1160 = reshape(shape = var_1159, x = var_1158)[name = tensor("op_1160")]; + tensor cross_9 = mul(x = var_1157, y = var_1160)[name = tensor("cross_9")]; + tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; + tensor var_1163 = const()[name = tensor("op_1163"), val = tensor([1, 1, 1, 1])]; + tensor var_1164 = reshape(shape = var_1163, x = valid_mask)[name = tensor("op_1164")]; + tensor v_masked_1 = mul(x = v_9, y = var_1164)[name = tensor("v_masked_1")]; + tensor var_1166 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1166")]; + tensor var_1168_transpose_x_1 = const()[name = tensor("op_1168_transpose_x_1"), val = tensor(true)]; + tensor var_1168_transpose_y_1 = const()[name = tensor("op_1168_transpose_y_1"), val = tensor(false)]; + tensor var_1168 = matmul(transpose_x = var_1168_transpose_x_1, transpose_y = var_1168_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1168")]; + tensor new_kv_unnorm_9 = add(x = var_1166, y = var_1168)[name = tensor("new_kv_unnorm_9")]; + tensor var_1170_keep_dims_0 = const()[name = tensor("op_1170_keep_dims_0"), val = tensor(false)]; + tensor var_1170 = reduce_sum(keep_dims = var_1170_keep_dims_0, x = valid_mask)[name = tensor("op_1170")]; + tensor var_1171 = const()[name = tensor("op_1171"), val = tensor([1])]; + tensor var_1172 = reshape(shape = var_1171, x = var_1170)[name = tensor("op_1172")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1172)[name = tensor("new_scale_9")]; tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; - tensor clip_10 = clip(alpha = var_1026, beta = const_21, x = new_scale_9)[name = tensor("clip_10")]; + tensor clip_10 = clip(alpha = var_1029, beta = const_21, x = new_scale_9)[name = tensor("clip_10")]; tensor sqrt_new_scale_1 = sqrt(x = clip_10)[name = tensor("sqrt_new_scale_1")]; - tensor var_1172 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1172")]; - tensor var_1173_perm_0 = const()[name = tensor("op_1173_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1176 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1176")]; + tensor var_1177_perm_0 = const()[name = tensor("op_1177_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_1173 = transpose(perm = var_1173_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1023, x = var_1173)[name = tensor("out_27")]; - tensor var_1177 = const()[name = tensor("op_1177"), val = tensor([12, 1, 256])]; - tensor out_29 = reshape(shape = var_1177, x = out_27)[name = tensor("out_29")]; - tensor var_1179 = silu(x = input_169)[name = tensor("op_1179")]; - tensor input_171 = mul(x = var_1179, y = out_29)[name = tensor("input_171")]; + tensor var_1177 = transpose(perm = var_1177_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1026, x = var_1177)[name = tensor("out_27")]; + tensor var_1181 = const()[name = tensor("op_1181"), val = tensor([12, 1, 256])]; + tensor out_29 = reshape(shape = var_1181, x = out_27)[name = tensor("out_29")]; + tensor var_1183 = silu(x = input_169)[name = tensor("op_1183")]; + tensor input_171 = mul(x = var_1183, y = out_29)[name = tensor("input_171")]; tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1021, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; - tensor var_1189 = const()[name = tensor("op_1189"), val = tensor([1, 12, 1, 256])]; - tensor var_1190 = reshape(shape = var_1189, x = xt_1)[name = tensor("op_1190")]; - tensor var_1191_perm_0 = const()[name = tensor("op_1191_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1194 = const()[name = tensor("op_1194"), val = tensor([1, 12, 256])]; - tensor var_1191 = transpose(perm = var_1191_perm_0, x = var_1190)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_1194, x = var_1191)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1024, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1193 = const()[name = tensor("op_1193"), val = tensor([1, 12, 1, 256])]; + tensor var_1194 = reshape(shape = var_1193, x = xt_1)[name = tensor("op_1194")]; + tensor var_1195_perm_0 = const()[name = tensor("op_1195_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1198 = const()[name = tensor("op_1198"), val = tensor([1, 12, 256])]; + tensor var_1195 = transpose(perm = var_1195_perm_0, x = var_1194)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1198, x = var_1195)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_1217 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_4 = const()[name = tensor("concat_4"), val = tensor([12, 1, 3, 256])]; - tensor var_1219 = reshape(shape = concat_4, x = var_1217)[name = tensor("op_1219")]; - tensor var_1220_axes_0 = const()[name = tensor("op_1220_axes_0"), val = tensor([0])]; - tensor var_1220 = expand_dims(axes = var_1220_axes_0, x = var_1219)[name = tensor("op_1220")]; - tensor var_1221_perm_0 = const()[name = tensor("op_1221_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1222_axes_0 = const()[name = tensor("op_1222_axes_0"), val = tensor([-2])]; - tensor var_1221 = transpose(perm = var_1221_perm_0, x = var_1220)[name = tensor("transpose_21")]; - tensor var_1222 = squeeze(axes = var_1222_axes_0, x = var_1221)[name = tensor("op_1222")]; + tensor var_1221 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_5 = const()[name = tensor("concat_5"), val = tensor([12, 1, 3, 256])]; + tensor var_1223 = reshape(shape = concat_5, x = var_1221)[name = tensor("op_1223")]; + tensor var_1224_axes_0 = const()[name = tensor("op_1224_axes_0"), val = tensor([0])]; + tensor var_1224 = expand_dims(axes = var_1224_axes_0, x = var_1223)[name = tensor("op_1224")]; + tensor var_1225_perm_0 = const()[name = tensor("op_1225_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1226_axes_0 = const()[name = tensor("op_1226_axes_0"), val = tensor([-2])]; + tensor var_1225 = transpose(perm = var_1225_perm_0, x = var_1224)[name = tensor("transpose_21")]; + tensor var_1226 = squeeze(axes = var_1226_axes_0, x = var_1225)[name = tensor("op_1226")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 12, 1, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1222)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1226)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 12, 1, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1222)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1226)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 12, 1, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1222)[name = tensor("v_11")]; - tensor var_1230 = const()[name = tensor("op_1230"), val = tensor([12, 4, 64])]; - tensor var_1231 = reshape(shape = var_1230, x = q_11)[name = tensor("op_1231")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1226)[name = tensor("v_11")]; + tensor var_1234 = const()[name = tensor("op_1234"), val = tensor([12, 4, 64])]; + tensor var_1235 = reshape(shape = var_1234, x = q_11)[name = tensor("op_1235")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1237 = const()[name = tensor("op_1237"), val = tensor([12, 4, 64])]; - tensor var_1238 = reshape(shape = var_1237, x = k_11)[name = tensor("op_1238")]; + tensor var_1241 = const()[name = tensor("op_1241"), val = tensor([12, 4, 64])]; + tensor var_1242 = reshape(shape = var_1241, x = k_11)[name = tensor("op_1242")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1244 = const()[name = tensor("op_1244"), val = tensor([12, 4, 64])]; - tensor var_1245 = reshape(shape = var_1244, x = v_11)[name = tensor("op_1245")]; + tensor var_1248 = const()[name = tensor("op_1248"), val = tensor([12, 4, 64])]; + tensor var_1249 = reshape(shape = var_1248, x = v_11)[name = tensor("op_1249")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1248 = const()[name = tensor("op_1248"), val = tensor([1, 4, 12, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_1231)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_1248, x = q_13)[name = tensor("q_15")]; - tensor var_1250 = const()[name = tensor("op_1250"), val = tensor([1, 4, 12, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_1238)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_1250, x = k_13)[name = tensor("k_15")]; tensor var_1252 = const()[name = tensor("op_1252"), val = tensor([1, 4, 12, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_1245)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_1252, x = v_13)[name = tensor("v_15")]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1235)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1252, x = q_13)[name = tensor("q_15")]; + tensor var_1254 = const()[name = tensor("op_1254"), val = tensor([1, 4, 12, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1242)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1254, x = k_13)[name = tensor("k_15")]; + tensor var_1256 = const()[name = tensor("op_1256"), val = tensor([1, 4, 12, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1249)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1256, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -1095,30 +1104,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_1255 = const()[name = tensor("op_1255"), val = tensor([2, 0, 1, 3])]; - tensor var_1260 = const()[name = tensor("op_1260"), val = tensor([12, 256])]; - tensor var_1256 = transpose(perm = var_1255, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_1260, x = var_1256)[name = tensor("attn_output_3")]; + tensor var_1259 = const()[name = tensor("op_1259"), val = tensor([2, 0, 1, 3])]; + tensor var_1264 = const()[name = tensor("op_1264"), val = tensor([12, 256])]; + tensor var_1260 = transpose(perm = var_1259, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1264, x = var_1260)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_1264 = const()[name = tensor("op_1264"), val = tensor([12, 1, 256])]; - tensor attn_output_7 = reshape(shape = var_1264, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1268 = const()[name = tensor("op_1268"), val = tensor([12, 1, 256])]; + tensor attn_output_7 = reshape(shape = var_1268, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; - tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1021, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1024, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1021, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; - tensor var_1284 = const()[name = tensor("op_1284"), val = tensor([1, 1, 12, 256])]; - tensor x_31 = reshape(shape = var_1284, x = xt_3)[name = tensor("x_31")]; - tensor var_1286_perm_0 = const()[name = tensor("op_1286_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1290 = const()[name = tensor("op_1290"), val = tensor([12, 1, 256])]; - tensor var_1286 = transpose(perm = var_1286_perm_0, x = x_31)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_1290, x = var_1286)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1024, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1288 = const()[name = tensor("op_1288"), val = tensor([1, 1, 12, 256])]; + tensor x_31 = reshape(shape = var_1288, x = xt_3)[name = tensor("x_31")]; + tensor var_1290_perm_0 = const()[name = tensor("op_1290_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1294 = const()[name = tensor("op_1294"), val = tensor([12, 1, 256])]; + tensor var_1290 = transpose(perm = var_1290_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1294, x = var_1290)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 12, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1129,120 +1138,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_1298 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_1299 = const()[name = tensor("op_1299"), val = tensor([12, 1, 4, 64])]; - tensor var_1300 = reshape(shape = var_1299, x = var_1298)[name = tensor("op_1300")]; + tensor var_1302 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1303 = const()[name = tensor("op_1303"), val = tensor([12, 1, 4, 64])]; + tensor var_1304 = reshape(shape = var_1303, x = var_1302)[name = tensor("op_1304")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1304 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_1305 = const()[name = tensor("op_1305"), val = tensor(0x1p-3)]; - tensor var_1306 = mul(x = var_1304, y = var_1305)[name = tensor("op_1306")]; - tensor var_1307 = const()[name = tensor("op_1307"), val = tensor([12, 1, 4, 64])]; - tensor var_1308 = reshape(shape = var_1307, x = var_1306)[name = tensor("op_1308")]; + tensor var_1308 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1309 = const()[name = tensor("op_1309"), val = tensor(0x1p-3)]; + tensor var_1310 = mul(x = var_1308, y = var_1309)[name = tensor("op_1310")]; + tensor var_1311 = const()[name = tensor("op_1311"), val = tensor([12, 1, 4, 64])]; + tensor var_1312 = reshape(shape = var_1311, x = var_1310)[name = tensor("op_1312")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1312 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_1313 = const()[name = tensor("op_1313"), val = tensor([12, 1, 4, 64])]; - tensor var_1314 = reshape(shape = var_1313, x = var_1312)[name = tensor("op_1314")]; + tensor var_1316 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1317 = const()[name = tensor("op_1317"), val = tensor([12, 1, 4, 64])]; + tensor var_1318 = reshape(shape = var_1317, x = var_1316)[name = tensor("op_1318")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; - tensor clip_11 = clip(alpha = var_1026, beta = const_32, x = s_t)[name = tensor("clip_11")]; + tensor clip_11 = clip(alpha = var_1029, beta = const_32, x = s_t)[name = tensor("clip_11")]; tensor sqrt_s_t = sqrt(x = clip_11)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_1308)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_1300)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1312)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1304)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_1329 = const()[name = tensor("op_1329"), val = tensor([1, 1])]; - tensor var_1330 = reshape(shape = var_1329, x = sqrt_s_t)[name = tensor("op_1330")]; - tensor M = real_div(x = var_1146, y = var_1330)[name = tensor("M")]; - tensor var_1332 = mul(x = qk, y = M)[name = tensor("op_1332")]; + tensor var_1333 = const()[name = tensor("op_1333"), val = tensor([1, 1])]; + tensor var_1334 = reshape(shape = var_1333, x = sqrt_s_t)[name = tensor("op_1334")]; + tensor M = real_div(x = var_1150, y = var_1334)[name = tensor("M")]; + tensor var_1336 = mul(x = qk, y = M)[name = tensor("op_1336")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_1314)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1332, y = v_17)[name = tensor("inner")]; - tensor var_1334_transpose_x_0 = const()[name = tensor("op_1334_transpose_x_0"), val = tensor(false)]; - tensor var_1334_transpose_y_0 = const()[name = tensor("op_1334_transpose_y_0"), val = tensor(false)]; - tensor var_1334 = matmul(transpose_x = var_1334_transpose_x_0, transpose_y = var_1334_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1334")]; - tensor var_1335 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1335")]; - tensor var_1336 = const()[name = tensor("op_1336"), val = tensor([1, 1, 1, 1])]; - tensor var_1337 = reshape(shape = var_1336, x = var_1335)[name = tensor("op_1337")]; - tensor cross = mul(x = var_1334, y = var_1337)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1318)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1336, y = v_17)[name = tensor("inner")]; + tensor var_1338_transpose_x_0 = const()[name = tensor("op_1338_transpose_x_0"), val = tensor(false)]; + tensor var_1338_transpose_y_0 = const()[name = tensor("op_1338_transpose_y_0"), val = tensor(false)]; + tensor var_1338 = matmul(transpose_x = var_1338_transpose_x_0, transpose_y = var_1338_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1338")]; + tensor var_1339 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1339")]; + tensor var_1340 = const()[name = tensor("op_1340"), val = tensor([1, 1, 1, 1])]; + tensor var_1341 = reshape(shape = var_1340, x = var_1339)[name = tensor("op_1341")]; + tensor cross = mul(x = var_1338, y = var_1341)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_1160)[name = tensor("v_masked")]; - tensor var_1343 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1343")]; - tensor var_1345_transpose_x_1 = const()[name = tensor("op_1345_transpose_x_1"), val = tensor(true)]; - tensor var_1345_transpose_y_1 = const()[name = tensor("op_1345_transpose_y_1"), val = tensor(false)]; - tensor var_1345 = matmul(transpose_x = var_1345_transpose_x_1, transpose_y = var_1345_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1345")]; - tensor new_kv_unnorm = add(x = var_1343, y = var_1345)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_1168)[name = tensor("new_scale")]; + tensor v_masked = mul(x = v_17, y = var_1164)[name = tensor("v_masked")]; + tensor var_1347 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1347")]; + tensor var_1349_transpose_x_1 = const()[name = tensor("op_1349_transpose_x_1"), val = tensor(true)]; + tensor var_1349_transpose_y_1 = const()[name = tensor("op_1349_transpose_y_1"), val = tensor(false)]; + tensor var_1349 = matmul(transpose_x = var_1349_transpose_x_1, transpose_y = var_1349_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1349")]; + tensor new_kv_unnorm = add(x = var_1347, y = var_1349)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1172)[name = tensor("new_scale")]; tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; - tensor clip_12 = clip(alpha = var_1026, beta = const_33, x = new_scale)[name = tensor("clip_12")]; + tensor clip_12 = clip(alpha = var_1029, beta = const_33, x = new_scale)[name = tensor("clip_12")]; tensor sqrt_new_scale = sqrt(x = clip_12)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_1354_perm_0 = const()[name = tensor("op_1354_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1358_perm_0 = const()[name = tensor("op_1358_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_1354 = transpose(perm = var_1354_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1023, x = var_1354)[name = tensor("out_33")]; - tensor var_1358 = const()[name = tensor("op_1358"), val = tensor([12, 1, 256])]; - tensor out = reshape(shape = var_1358, x = out_33)[name = tensor("out")]; - tensor var_1360 = silu(x = input_187)[name = tensor("op_1360")]; - tensor input_189 = mul(x = var_1360, y = out)[name = tensor("input_189")]; + tensor var_1358 = transpose(perm = var_1358_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1026, x = var_1358)[name = tensor("out_33")]; + tensor var_1362 = const()[name = tensor("op_1362"), val = tensor([12, 1, 256])]; + tensor out = reshape(shape = var_1362, x = out_33)[name = tensor("out")]; + tensor var_1364 = silu(x = input_187)[name = tensor("op_1364")]; + tensor input_189 = mul(x = var_1364, y = out)[name = tensor("input_189")]; tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1021, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; - tensor var_1370 = const()[name = tensor("op_1370"), val = tensor([1, 12, 1, 256])]; - tensor var_1371 = reshape(shape = var_1370, x = xt_5)[name = tensor("op_1371")]; - tensor var_1372_perm_0 = const()[name = tensor("op_1372_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1375 = const()[name = tensor("op_1375"), val = tensor([1, 12, 256])]; - tensor var_1372 = transpose(perm = var_1372_perm_0, x = var_1371)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_1375, x = var_1372)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1024, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1374 = const()[name = tensor("op_1374"), val = tensor([1, 12, 1, 256])]; + tensor var_1375 = reshape(shape = var_1374, x = xt_5)[name = tensor("op_1375")]; + tensor var_1376_perm_0 = const()[name = tensor("op_1376_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1379 = const()[name = tensor("op_1379"), val = tensor([1, 12, 256])]; + tensor var_1376 = transpose(perm = var_1376_perm_0, x = var_1375)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1379, x = var_1376)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_1398 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_5 = const()[name = tensor("concat_5"), val = tensor([12, 1, 3, 256])]; - tensor var_1400 = reshape(shape = concat_5, x = var_1398)[name = tensor("op_1400")]; - tensor var_1401_axes_0 = const()[name = tensor("op_1401_axes_0"), val = tensor([0])]; - tensor var_1401 = expand_dims(axes = var_1401_axes_0, x = var_1400)[name = tensor("op_1401")]; - tensor var_1402_perm_0 = const()[name = tensor("op_1402_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1403_axes_0 = const()[name = tensor("op_1403_axes_0"), val = tensor([-2])]; - tensor var_1402 = transpose(perm = var_1402_perm_0, x = var_1401)[name = tensor("transpose_8")]; - tensor var_1403 = squeeze(axes = var_1403_axes_0, x = var_1402)[name = tensor("op_1403")]; + tensor var_1402 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_6 = const()[name = tensor("concat_6"), val = tensor([12, 1, 3, 256])]; + tensor var_1404 = reshape(shape = concat_6, x = var_1402)[name = tensor("op_1404")]; + tensor var_1405_axes_0 = const()[name = tensor("op_1405_axes_0"), val = tensor([0])]; + tensor var_1405 = expand_dims(axes = var_1405_axes_0, x = var_1404)[name = tensor("op_1405")]; + tensor var_1406_perm_0 = const()[name = tensor("op_1406_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1407_axes_0 = const()[name = tensor("op_1407_axes_0"), val = tensor([-2])]; + tensor var_1406 = transpose(perm = var_1406_perm_0, x = var_1405)[name = tensor("transpose_8")]; + tensor var_1407 = squeeze(axes = var_1407_axes_0, x = var_1406)[name = tensor("op_1407")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 12, 1, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1403)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1407)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 12, 1, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1403)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1407)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 12, 1, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1403)[name = tensor("v_19")]; - tensor var_1411 = const()[name = tensor("op_1411"), val = tensor([12, 4, 64])]; - tensor var_1412 = reshape(shape = var_1411, x = q_19)[name = tensor("op_1412")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1407)[name = tensor("v_19")]; + tensor var_1415 = const()[name = tensor("op_1415"), val = tensor([12, 4, 64])]; + tensor var_1416 = reshape(shape = var_1415, x = q_19)[name = tensor("op_1416")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1418 = const()[name = tensor("op_1418"), val = tensor([12, 4, 64])]; - tensor var_1419 = reshape(shape = var_1418, x = k_19)[name = tensor("op_1419")]; + tensor var_1422 = const()[name = tensor("op_1422"), val = tensor([12, 4, 64])]; + tensor var_1423 = reshape(shape = var_1422, x = k_19)[name = tensor("op_1423")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1425 = const()[name = tensor("op_1425"), val = tensor([12, 4, 64])]; - tensor var_1426 = reshape(shape = var_1425, x = v_19)[name = tensor("op_1426")]; + tensor var_1429 = const()[name = tensor("op_1429"), val = tensor([12, 4, 64])]; + tensor var_1430 = reshape(shape = var_1429, x = v_19)[name = tensor("op_1430")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1429 = const()[name = tensor("op_1429"), val = tensor([1, 4, 12, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_1412)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_1429, x = q_21)[name = tensor("q")]; - tensor var_1431 = const()[name = tensor("op_1431"), val = tensor([1, 4, 12, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_1419)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_1431, x = k_21)[name = tensor("k")]; tensor var_1433 = const()[name = tensor("op_1433"), val = tensor([1, 4, 12, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_1426)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_1433, x = v_21)[name = tensor("v")]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1416)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1433, x = q_21)[name = tensor("q")]; + tensor var_1435 = const()[name = tensor("op_1435"), val = tensor([1, 4, 12, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1423)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1435, x = k_21)[name = tensor("k")]; + tensor var_1437 = const()[name = tensor("op_1437"), val = tensor([1, 4, 12, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1430)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1437, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -1253,35 +1262,35 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_1436 = const()[name = tensor("op_1436"), val = tensor([2, 0, 1, 3])]; - tensor var_1441 = const()[name = tensor("op_1441"), val = tensor([12, 256])]; - tensor var_1437 = transpose(perm = var_1436, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_1441, x = var_1437)[name = tensor("attn_output_11")]; + tensor var_1440 = const()[name = tensor("op_1440"), val = tensor([2, 0, 1, 3])]; + tensor var_1445 = const()[name = tensor("op_1445"), val = tensor([12, 256])]; + tensor var_1441 = transpose(perm = var_1440, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1445, x = var_1441)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_1445 = const()[name = tensor("op_1445"), val = tensor([12, 1, 256])]; - tensor attn_output = reshape(shape = var_1445, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1449 = const()[name = tensor("op_1449"), val = tensor([12, 1, 256])]; + tensor attn_output = reshape(shape = var_1449, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; - tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1021, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1024, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1021, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; - tensor var_1465 = const()[name = tensor("op_1465"), val = tensor([1, 1, 12, 256])]; - tensor input = reshape(shape = var_1465, x = xt)[name = tensor("input")]; - tensor var_1467 = const()[name = tensor("op_1467"), val = tensor([-1])]; - tensor var_1468 = reduce_l2_norm(axes = var_1467, keep_dims = var_1024, x = input)[name = tensor("op_1468")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1024, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1469 = const()[name = tensor("op_1469"), val = tensor([1, 1, 12, 256])]; + tensor input = reshape(shape = var_1469, x = xt)[name = tensor("input")]; + tensor var_1471 = const()[name = tensor("op_1471"), val = tensor([-1])]; + tensor var_1472 = reduce_l2_norm(axes = var_1471, keep_dims = var_1027, x = input)[name = tensor("op_1472")]; tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; - tensor clip_13 = clip(alpha = var_1015, beta = const_42, x = var_1468)[name = tensor("clip_13")]; - tensor var_1470 = real_div(x = input, y = clip_13)[name = tensor("op_1470")]; + tensor clip_13 = clip(alpha = var_1019, beta = const_42, x = var_1472)[name = tensor("clip_13")]; + tensor var_1474 = real_div(x = input, y = clip_13)[name = tensor("op_1474")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_10 = const()[name = tensor("concat_10"), val = tensor([1, 256, 12])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1470)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_10, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_11 = const()[name = tensor("concat_11"), val = tensor([1, 256, 12])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1474)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_11, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = emb, y = reshape_1)[name = tensor("matmul_2")]; @@ -1289,10 +1298,10 @@ program(1.0) tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 1, 11])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = matmul_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_1474")]; - tensor var_1476_axis_0 = const()[name = tensor("op_1476_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_1476_axis_0, values = (var_1172, nkv))[name = tensor("op_1476")]; - tensor var_1478_axis_0 = const()[name = tensor("op_1478_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_1478_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1478")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1478")]; + tensor var_1480_axis_0 = const()[name = tensor("op_1480_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1480_axis_0, values = (var_1176, nkv))[name = tensor("op_1480")]; + tensor var_1482_axis_0 = const()[name = tensor("op_1482_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1482_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1482")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/DIHARD III/ls_eend_dih3_100ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/DIHARD III/ls_eend_dih3_100ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index 153787e1ae396fb9ff83a82328bb55cc9887df3e..8898d947253325e6cfaf11364b8b86000c1612aa 100644 --- a/optimized/DIHARD III/ls_eend_dih3_100ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/DIHARD III/ls_eend_dih3_100ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b0f8e63d136110f46851871edda04a9cc6419f69c141597bfde1f86cf404fd39 -size 184737 +oid sha256:20d5b5c886833a6a568e5e7098d50cd19d16c4371c78702244e0bbacffec9c88 +size 185995 diff --git a/optimized/DIHARD III/ls_eend_dih3_100ms.mlpackage/Manifest.json b/optimized/DIHARD III/ls_eend_dih3_100ms.mlpackage/Manifest.json index ffa7c0a7effd809a47d698715c61a2bea6f32388..fb0b8a64f94e60877136dccf5858a209af98552a 100644 --- a/optimized/DIHARD III/ls_eend_dih3_100ms.mlpackage/Manifest.json +++ b/optimized/DIHARD III/ls_eend_dih3_100ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "5C90FB83-881E-47E3-8E2E-E693B86730C7": { + "1D5FEFF2-E4FE-48C5-8087-30569950F68E": { "author": "com.apple.CoreML", "description": "CoreML Model Specification", "name": "model.mlmodel", "path": "com.apple.CoreML/model.mlmodel" }, - "B389B826-C79F-48B7-B5DC-51184B74427C": { + "30D6D0A4-611E-4D1B-B038-6F60570213DE": { "author": "com.apple.CoreML", "description": "CoreML Model Weights", "name": "weights", "path": "com.apple.CoreML/weights" } }, - "rootModelIdentifier": "5C90FB83-881E-47E3-8E2E-E693B86730C7" + "rootModelIdentifier": "1D5FEFF2-E4FE-48C5-8087-30569950F68E" } diff --git a/optimized/DIHARD III/ls_eend_dih3_200ms.mlmodelc/analytics/coremldata.bin b/optimized/DIHARD III/ls_eend_dih3_200ms.mlmodelc/analytics/coremldata.bin index 221186196ba1d0d37f79301ded70f7e5fce48d23..a1fb3291a59a6ba998978bde6c8d9361cb5a73c8 100644 --- a/optimized/DIHARD III/ls_eend_dih3_200ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/DIHARD III/ls_eend_dih3_200ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:482a4d21f673bdb5f03c68c836eabb41c258e74522ed113c658a44c57ecbb53f +oid sha256:950327435363ae304a408af6ab12658c23e68fb33d9358d8d89856cd3296faea size 243 diff --git a/optimized/DIHARD III/ls_eend_dih3_200ms.mlmodelc/coremldata.bin b/optimized/DIHARD III/ls_eend_dih3_200ms.mlmodelc/coremldata.bin index 342fa980dce29a131408a507fef7498ba08c08e4..1d1a69b3e6ff9911ce5b3f0322fe313a29978bbb 100644 --- a/optimized/DIHARD III/ls_eend_dih3_200ms.mlmodelc/coremldata.bin +++ b/optimized/DIHARD III/ls_eend_dih3_200ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ef6d5f1ac258b6b90c7659ebb07a70481b18d0822aa94387c9287040d82b879a -size 1268 +oid sha256:2a1fb2fc9fb09e9d08f693e5f124f10ce6e77c19c363ebbd5db1f4857b8febd3 +size 1359 diff --git a/optimized/DIHARD III/ls_eend_dih3_200ms.mlmodelc/metadata.json b/optimized/DIHARD III/ls_eend_dih3_200ms.mlmodelc/metadata.json index 0fbb1b94a9bb9f947859c678c93d8027db6c902c..4ba72d2e09289916438f9ebc3d157eb3605405ce 100644 --- a/optimized/DIHARD III/ls_eend_dih3_200ms.mlmodelc/metadata.json +++ b/optimized/DIHARD III/ls_eend_dih3_200ms.mlmodelc/metadata.json @@ -96,30 +96,30 @@ "Ios17.reshape" : 84, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 61, - "Split" : 8, + "Ios17.transpose" : 57, + "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 70, "Tile" : 9, - "Ios17.sliceByIndex" : 60, - "Ios16.sigmoid" : 9, + "Ios17.sliceByIndex" : 52, + "Ios16.sigmoid" : 5, "Ios16.reduceSum" : 1, "Ios17.squeeze" : 2, - "Ios17.layerNorm" : 37, - "Ios17.batchNorm" : 8, + "Ios17.layerNorm" : 33, + "Ios17.batchNorm" : 4, "Ios17.cast" : 16, "Ios17.sqrt" : 18, "Ios17.sub" : 16, - "Ios17.conv" : 25, + "Ios17.conv" : 13, "Ios17.realDiv" : 20, "Ios17.linear" : 56, "Ios17.concat" : 14, "Ios17.clip" : 22, "Ios17.greaterEqual" : 8, - "Ios16.silu" : 22, + "Ios16.silu" : 18, "Stack" : 6, "Ios16.cumsum" : 1, - "Ios17.mul" : 83 + "Ios17.mul" : 79 }, "computePrecision" : "Mixed (Float32, Int32)", "isUpdatable" : "0", @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"dih3\", \"model_label\": \"DIHARD III\", \"variant\": \"pipeline\", \"T\": 2, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"dih3\", \"model_label\": \"DIHARD III\", \"variant\": \"pipeline\", \"chunk_size\": 2, \"step_duration_ms\": 200, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/DIHARD III/ls_eend_dih3_200ms.mlmodelc/model.mil b/optimized/DIHARD III/ls_eend_dih3_200ms.mlmodelc/model.mil index 114ace2e9611bac24b7643b50777d9d91061c4a9..1385f671a75237f030dc475fda5fd7ef583c51e3 100644 --- a/optimized/DIHARD III/ls_eend_dih3_200ms.mlmodelc/model.mil +++ b/optimized/DIHARD III/ls_eend_dih3_200ms.mlmodelc/model.mil @@ -217,6 +217,7 @@ program(1.0) tensor var_21 = const()[name = tensor("op_21"), val = tensor(0x0p+0)]; tensor var_24 = const()[name = tensor("op_24"), val = tensor(0x1.0c6f7ap-20)]; tensor var_27 = const()[name = tensor("op_27"), val = tensor(2)]; + tensor var_30 = const()[name = tensor("op_30"), val = tensor(0)]; tensor var_33 = const()[name = tensor("op_33"), val = tensor(1)]; tensor var_35 = const()[name = tensor("op_35"), val = tensor(0x1.4f8b58p-17)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(true)]; @@ -263,8 +264,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_61")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_62")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_196 = const()[name = tensor("op_196"), val = tensor([2, 1])]; tensor var_197 = reshape(shape = var_196, x = sqrt_s_t_1)[name = tensor("op_197")]; @@ -272,7 +273,7 @@ program(1.0) tensor var_199 = mul(x = qk_1, y = M_1)[name = tensor("op_199")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_60")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_199, y = v_1)[name = tensor("inner_1")]; tensor var_201_transpose_x_0 = const()[name = tensor("op_201_transpose_x_0"), val = tensor(false)]; tensor var_201_transpose_y_0 = const()[name = tensor("op_201_transpose_y_0"), val = tensor(false)]; @@ -293,7 +294,7 @@ program(1.0) tensor var_214 = real_div(x = new_kv_unnorm_1, y = var_213)[name = tensor("op_214")]; tensor var_215_perm_0 = const()[name = tensor("op_215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_59")]; + tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_24, x = var_215)[name = tensor("out_3")]; tensor var_219 = const()[name = tensor("op_219"), val = tensor([1, 2, 256])]; tensor out_5 = reshape(shape = var_219, x = out_3)[name = tensor("out_5")]; @@ -341,145 +342,108 @@ program(1.0) tensor var_252 = sub(x = var_17, y = use_roll_1)[name = tensor("op_252")]; tensor var_253 = mul(x = filled_window_1, y = var_252)[name = tensor("op_253")]; tensor var_254 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_254")]; - tensor input_21 = add(x = var_253, y = var_254)[name = tensor("input_21")]; + tensor window_3 = add(x = var_253, y = var_254)[name = tensor("window_3")]; tensor var_256 = const()[name = tensor("op_256"), val = tensor(0x1p+0)]; tensor var_257 = add(x = filled_1, y = var_256)[name = tensor("op_257")]; tensor clip_1 = clip(alpha = var_21, beta = var_16, x = var_257)[name = tensor("clip_1")]; + tensor var_260_begin_0 = const()[name = tensor("op_260_begin_0"), val = tensor([0, 1, 0])]; + tensor var_260_end_0 = const()[name = tensor("op_260_end_0"), val = tensor([1, 1, 256])]; + tensor var_260_end_mask_0 = const()[name = tensor("op_260_end_mask_0"), val = tensor([true, true, true])]; + tensor var_260 = slice_by_index(begin = var_260_begin_0, end = var_260_end_0, end_mask = var_260_end_mask_0, x = x_3)[name = tensor("op_260")]; + tensor var_263_begin_0 = const()[name = tensor("op_263_begin_0"), val = tensor([0, 1, 0])]; + tensor var_263_end_0 = const()[name = tensor("op_263_end_0"), val = tensor([1, 16, 256])]; + tensor var_263_end_mask_0 = const()[name = tensor("op_263_end_mask_0"), val = tensor([true, true, true])]; + tensor var_263 = slice_by_index(begin = var_263_begin_0, end = var_263_end_0, end_mask = var_263_end_mask_0, x = window_3)[name = tensor("op_263")]; + tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; + tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_263, var_260))[name = tensor("rolled_3")]; + tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; + tensor var_268 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_268")]; + tensor var_269_dtype_0 = const()[name = tensor("op_269_dtype_0"), val = tensor("fp32")]; + tensor var_270 = const()[name = tensor("op_270"), val = tensor([1, 16, 1])]; + tensor var_269 = cast(dtype = var_269_dtype_0, x = var_268)[name = tensor("cast_141")]; + tensor fill_mask_3 = reshape(shape = var_270, x = var_269)[name = tensor("fill_mask_3")]; + tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_260)[name = tensor("x_expanded_3")]; + tensor var_274 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_274")]; + tensor var_275 = mul(x = window_3, y = var_274)[name = tensor("op_275")]; + tensor var_276 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_276")]; + tensor filled_window_3 = add(x = var_275, y = var_276)[name = tensor("filled_window_3")]; + tensor var_278 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_278")]; + tensor var_279_dtype_0 = const()[name = tensor("op_279_dtype_0"), val = tensor("fp32")]; + tensor var_280 = const()[name = tensor("op_280"), val = tensor([1, 1, 1])]; + tensor var_279 = cast(dtype = var_279_dtype_0, x = var_278)[name = tensor("cast_140")]; + tensor use_roll_3 = reshape(shape = var_280, x = var_279)[name = tensor("use_roll_3")]; + tensor var_282 = sub(x = var_17, y = use_roll_3)[name = tensor("op_282")]; + tensor var_283 = mul(x = filled_window_3, y = var_282)[name = tensor("op_283")]; + tensor var_284 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_284")]; + tensor window_5 = add(x = var_283, y = var_284)[name = tensor("window_5")]; + tensor var_286 = const()[name = tensor("op_286"), val = tensor(0x1p+0)]; + tensor var_287 = add(x = clip_1, y = var_286)[name = tensor("op_287")]; + tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_287)[name = tensor("clip_3")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_30, interleave = input_21_interleave_0, values = (window_3, window_5))[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; - tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; + tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; tensor inputs_3_pad_type_0 = const()[name = tensor("inputs_3_pad_type_0"), val = tensor("valid")]; tensor inputs_3_strides_0 = const()[name = tensor("inputs_3_strides_0"), val = tensor([1])]; tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_58")]; - tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_278_split_sizes_0 = const()[name = tensor("op_278_split_sizes_0"), val = tensor([256, 256])]; - tensor var_278_axis_0 = const()[name = tensor("op_278_axis_0"), val = tensor(1)]; - tensor var_278_0, tensor var_278_1 = split(axis = var_278_axis_0, split_sizes = var_278_split_sizes_0, x = inputs_3)[name = tensor("op_278")]; - tensor var_280 = sigmoid(x = var_278_1)[name = tensor("op_280")]; - tensor inputs_5 = mul(x = var_278_0, y = var_280)[name = tensor("inputs_5")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; + tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; + tensor var_310_split_sizes_0 = const()[name = tensor("op_310_split_sizes_0"), val = tensor([256, 256])]; + tensor var_310_axis_0 = const()[name = tensor("op_310_axis_0"), val = tensor(1)]; + tensor var_310_0, tensor var_310_1 = split(axis = var_310_axis_0, split_sizes = var_310_split_sizes_0, x = inputs_3)[name = tensor("op_310")]; + tensor var_312 = sigmoid(x = var_310_1)[name = tensor("op_312")]; + tensor inputs_5 = mul(x = var_310_0, y = var_312)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; tensor outputs_aug_1_strides_0 = const()[name = tensor("outputs_aug_1_strides_0"), val = tensor([1])]; tensor outputs_aug_1_dilations_0 = const()[name = tensor("outputs_aug_1_dilations_0"), val = tensor([1])]; - tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; + tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; tensor input_25_begin_0 = const()[name = tensor("input_25_begin_0"), val = tensor([0, 0, 0])]; - tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([1, 256, 16])]; + tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([2, 256, 16])]; tensor input_25_end_mask_0 = const()[name = tensor("input_25_end_mask_0"), val = tensor([true, true, false])]; - tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; - tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; - tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; + tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; + tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; + tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; tensor input_29_pad_type_0 = const()[name = tensor("input_29_pad_type_0"), val = tensor("valid")]; tensor input_29_strides_0 = const()[name = tensor("input_29_strides_0"), val = tensor([1])]; tensor input_29_pad_0 = const()[name = tensor("input_29_pad_0"), val = tensor([0, 0])]; tensor input_29_dilations_0 = const()[name = tensor("input_29_dilations_0"), val = tensor([1])]; tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; - tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; + tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_311_begin_0 = const()[name = tensor("op_311_begin_0"), val = tensor([0, -1, 0])]; - tensor var_311_end_0 = const()[name = tensor("op_311_end_0"), val = tensor([1, 16, 256])]; - tensor var_311_end_mask_0 = const()[name = tensor("op_311_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_57")]; - tensor var_311 = slice_by_index(begin = var_311_begin_0, end = var_311_end_0, end_mask = var_311_end_mask_0, x = conv_out_1)[name = tensor("op_311")]; - tensor var_314_begin_0 = const()[name = tensor("op_314_begin_0"), val = tensor([0, 1, 0])]; - tensor var_314_end_0 = const()[name = tensor("op_314_end_0"), val = tensor([1, 1, 256])]; - tensor var_314_end_mask_0 = const()[name = tensor("op_314_end_mask_0"), val = tensor([true, true, true])]; - tensor var_314 = slice_by_index(begin = var_314_begin_0, end = var_314_end_0, end_mask = var_314_end_mask_0, x = x_3)[name = tensor("op_314")]; - tensor var_317_begin_0 = const()[name = tensor("op_317_begin_0"), val = tensor([0, 1, 0])]; - tensor var_317_end_0 = const()[name = tensor("op_317_end_0"), val = tensor([1, 16, 256])]; - tensor var_317_end_mask_0 = const()[name = tensor("op_317_end_mask_0"), val = tensor([true, true, true])]; - tensor var_317 = slice_by_index(begin = var_317_begin_0, end = var_317_end_0, end_mask = var_317_end_mask_0, x = input_21)[name = tensor("op_317")]; - tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_317, var_314))[name = tensor("rolled_3")]; - tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; - tensor var_322 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_322")]; - tensor var_323_dtype_0 = const()[name = tensor("op_323_dtype_0"), val = tensor("fp32")]; - tensor var_324 = const()[name = tensor("op_324"), val = tensor([1, 16, 1])]; - tensor var_323 = cast(dtype = var_323_dtype_0, x = var_322)[name = tensor("cast_141")]; - tensor fill_mask_3 = reshape(shape = var_324, x = var_323)[name = tensor("fill_mask_3")]; - tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_314)[name = tensor("x_expanded_3")]; - tensor var_328 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_328")]; - tensor var_329 = mul(x = input_21, y = var_328)[name = tensor("op_329")]; - tensor var_330 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_330")]; - tensor filled_window_3 = add(x = var_329, y = var_330)[name = tensor("filled_window_3")]; - tensor var_332 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_332")]; - tensor var_333_dtype_0 = const()[name = tensor("op_333_dtype_0"), val = tensor("fp32")]; - tensor var_334 = const()[name = tensor("op_334"), val = tensor([1, 1, 1])]; - tensor var_333 = cast(dtype = var_333_dtype_0, x = var_332)[name = tensor("cast_140")]; - tensor use_roll_3 = reshape(shape = var_334, x = var_333)[name = tensor("use_roll_3")]; - tensor var_336 = sub(x = var_17, y = use_roll_3)[name = tensor("op_336")]; - tensor var_337 = mul(x = filled_window_3, y = var_336)[name = tensor("op_337")]; - tensor var_338 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_338")]; - tensor input_31 = add(x = var_337, y = var_338)[name = tensor("input_31")]; - tensor var_340 = const()[name = tensor("op_340"), val = tensor(0x1p+0)]; - tensor var_341 = add(x = clip_1, y = var_340)[name = tensor("op_341")]; - tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_341)[name = tensor("clip_3")]; - tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; - tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_31)[name = tensor("x_7")]; - tensor input_33_perm_0 = const()[name = tensor("input_33_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_9_pad_type_0 = const()[name = tensor("inputs_9_pad_type_0"), val = tensor("valid")]; - tensor inputs_9_strides_0 = const()[name = tensor("inputs_9_strides_0"), val = tensor([1])]; - tensor inputs_9_pad_0 = const()[name = tensor("inputs_9_pad_0"), val = tensor([0, 0])]; - tensor inputs_9_dilations_0 = const()[name = tensor("inputs_9_dilations_0"), val = tensor([1])]; - tensor inputs_9_groups_0 = const()[name = tensor("inputs_9_groups_0"), val = tensor(1)]; - tensor input_33 = transpose(perm = input_33_perm_0, x = x_7)[name = tensor("transpose_56")]; - tensor inputs_9 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_9_dilations_0, groups = inputs_9_groups_0, pad = inputs_9_pad_0, pad_type = inputs_9_pad_type_0, strides = inputs_9_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_33)[name = tensor("inputs_9")]; - tensor var_362_split_sizes_0 = const()[name = tensor("op_362_split_sizes_0"), val = tensor([256, 256])]; - tensor var_362_axis_0 = const()[name = tensor("op_362_axis_0"), val = tensor(1)]; - tensor var_362_0, tensor var_362_1 = split(axis = var_362_axis_0, split_sizes = var_362_split_sizes_0, x = inputs_9)[name = tensor("op_362")]; - tensor var_364 = sigmoid(x = var_362_1)[name = tensor("op_364")]; - tensor inputs_11 = mul(x = var_362_0, y = var_364)[name = tensor("inputs_11")]; - tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; - tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; - tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; - tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_11)[name = tensor("outputs_aug_3")]; - tensor input_35_begin_0 = const()[name = tensor("input_35_begin_0"), val = tensor([0, 0, 0])]; - tensor input_35_end_0 = const()[name = tensor("input_35_end_0"), val = tensor([1, 256, 16])]; - tensor input_35_end_mask_0 = const()[name = tensor("input_35_end_mask_0"), val = tensor([true, true, false])]; - tensor input_35 = slice_by_index(begin = input_35_begin_0, end = input_35_end_0, end_mask = input_35_end_mask_0, x = outputs_aug_3)[name = tensor("input_35")]; - tensor inputs_13 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_35)[name = tensor("inputs_13")]; - tensor input_37 = silu(x = inputs_13)[name = tensor("input_37")]; - tensor input_39_pad_type_0 = const()[name = tensor("input_39_pad_type_0"), val = tensor("valid")]; - tensor input_39_strides_0 = const()[name = tensor("input_39_strides_0"), val = tensor([1])]; - tensor input_39_pad_0 = const()[name = tensor("input_39_pad_0"), val = tensor([0, 0])]; - tensor input_39_dilations_0 = const()[name = tensor("input_39_dilations_0"), val = tensor([1])]; - tensor input_39_groups_0 = const()[name = tensor("input_39_groups_0"), val = tensor(1)]; - tensor input_39 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_37)[name = tensor("input_39")]; - tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_395_begin_0 = const()[name = tensor("op_395_begin_0"), val = tensor([0, -1, 0])]; - tensor var_395_end_0 = const()[name = tensor("op_395_end_0"), val = tensor([1, 16, 256])]; - tensor var_395_end_mask_0 = const()[name = tensor("op_395_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_39)[name = tensor("transpose_55")]; - tensor var_395 = slice_by_index(begin = var_395_begin_0, end = var_395_end_0, end_mask = var_395_end_mask_0, x = conv_out_3)[name = tensor("op_395")]; - tensor var_398_interleave_0 = const()[name = tensor("op_398_interleave_0"), val = tensor(false)]; - tensor var_398 = concat(axis = var_33, interleave = var_398_interleave_0, values = (var_311, var_395))[name = tensor("op_398")]; - tensor input_41 = add(x = x_3, y = var_398)[name = tensor("input_41")]; + tensor var_343_begin_0 = const()[name = tensor("op_343_begin_0"), val = tensor([0, -1, 0])]; + tensor var_343_end_0 = const()[name = tensor("op_343_end_0"), val = tensor([2, 16, 256])]; + tensor var_343_end_mask_0 = const()[name = tensor("op_343_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_343 = slice_by_index(begin = var_343_begin_0, end = var_343_end_0, end_mask = var_343_end_mask_0, x = conv_out_1)[name = tensor("op_343")]; + tensor var_345_perm_0 = const()[name = tensor("op_345_perm_0"), val = tensor([1, 0, 2])]; + tensor var_345 = transpose(perm = var_345_perm_0, x = var_343)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_345)[name = tensor("input_31")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; + tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; + tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; + tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; + tensor var_368 = const()[name = tensor("op_368"), val = tensor(0x1p-1)]; + tensor var_369 = mul(x = input_39, y = var_368)[name = tensor("op_369")]; + tensor input_41 = add(x = var_369, y = input_31)[name = tensor("input_41")]; tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; - tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_41)[name = tensor("input_43")]; - tensor inputs_15 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_43)[name = tensor("linear_8")]; - tensor input_45 = silu(x = inputs_15)[name = tensor("input_45")]; - tensor input_49 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_45)[name = tensor("linear_9")]; - tensor var_421 = const()[name = tensor("op_421"), val = tensor(0x1p-1)]; - tensor var_422 = mul(x = input_49, y = var_421)[name = tensor("op_422")]; - tensor input_51 = add(x = var_422, y = input_41)[name = tensor("input_51")]; - tensor input_53_axes_0 = const()[name = tensor("input_53_axes_0"), val = tensor([-1])]; - tensor input_53 = layer_norm(axes = input_53_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_51)[name = tensor("input_53")]; - tensor input_55_axes_0 = const()[name = tensor("input_55_axes_0"), val = tensor([-1])]; - tensor input_55 = layer_norm(axes = input_55_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_53)[name = tensor("input_55")]; - tensor inputs_17 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_55)[name = tensor("linear_10")]; - tensor input_57 = silu(x = inputs_17)[name = tensor("input_57")]; - tensor input_61 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_57)[name = tensor("linear_11")]; - tensor var_451 = const()[name = tensor("op_451"), val = tensor(0x1p-1)]; - tensor var_452 = mul(x = input_61, y = var_451)[name = tensor("op_452")]; - tensor input_63 = add(x = var_452, y = input_53)[name = tensor("input_63")]; - tensor x_9_axes_0 = const()[name = tensor("x_9_axes_0"), val = tensor([-1])]; - tensor x_9 = layer_norm(axes = x_9_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_63)[name = tensor("x_9")]; + tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; + tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; + tensor input_45 = layer_norm(axes = input_45_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_43)[name = tensor("input_45")]; + tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; + tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; + tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; + tensor var_398 = const()[name = tensor("op_398"), val = tensor(0x1p-1)]; + tensor var_399 = mul(x = input_51, y = var_398)[name = tensor("op_399")]; + tensor input_53 = add(x = var_399, y = input_43)[name = tensor("input_53")]; + tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; + tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_3_end_0 = const()[name = tensor("prev_kv_3_end_0"), val = tensor([2, 1, 4, 64, 64])]; tensor prev_kv_3_end_mask_0 = const()[name = tensor("prev_kv_3_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -490,243 +454,206 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_466 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_9)[name = tensor("linear_12")]; - tensor var_467 = const()[name = tensor("op_467"), val = tensor([1, 2, 4, 64])]; - tensor var_468 = reshape(shape = var_467, x = var_466)[name = tensor("op_468")]; + tensor var_413 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_414 = const()[name = tensor("op_414"), val = tensor([1, 2, 4, 64])]; + tensor var_415 = reshape(shape = var_414, x = var_413)[name = tensor("op_415")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_472 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_9)[name = tensor("linear_13")]; - tensor var_473 = const()[name = tensor("op_473"), val = tensor(0x1p-3)]; - tensor var_474 = mul(x = var_472, y = var_473)[name = tensor("op_474")]; - tensor var_475 = const()[name = tensor("op_475"), val = tensor([1, 2, 4, 64])]; - tensor var_476 = reshape(shape = var_475, x = var_474)[name = tensor("op_476")]; + tensor var_419 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_420 = const()[name = tensor("op_420"), val = tensor(0x1p-3)]; + tensor var_421 = mul(x = var_419, y = var_420)[name = tensor("op_421")]; + tensor var_422 = const()[name = tensor("op_422"), val = tensor([1, 2, 4, 64])]; + tensor var_423 = reshape(shape = var_422, x = var_421)[name = tensor("op_423")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_480 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_9)[name = tensor("linear_14")]; - tensor var_481 = const()[name = tensor("op_481"), val = tensor([1, 2, 4, 64])]; - tensor var_482 = reshape(shape = var_481, x = var_480)[name = tensor("op_482")]; + tensor var_427 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_428 = const()[name = tensor("op_428"), val = tensor([1, 2, 4, 64])]; + tensor var_429 = reshape(shape = var_428, x = var_427)[name = tensor("op_429")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_67 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_9)[name = tensor("linear_15")]; + tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; tensor s_t_3 = add(x = prev_scale_3, y = encoder__t_index)[name = tensor("s_t_3")]; tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_476)[name = tensor("transpose_53")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_468)[name = tensor("transpose_54")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_423)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_415)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_492 = const()[name = tensor("op_492"), val = tensor([2, 1])]; - tensor var_493 = reshape(shape = var_492, x = sqrt_s_t_3)[name = tensor("op_493")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_493)[name = tensor("M_3")]; - tensor var_495 = mul(x = qk_3, y = M_3)[name = tensor("op_495")]; + tensor var_439 = const()[name = tensor("op_439"), val = tensor([2, 1])]; + tensor var_440 = reshape(shape = var_439, x = sqrt_s_t_3)[name = tensor("op_440")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_440)[name = tensor("M_3")]; + tensor var_442 = mul(x = qk_3, y = M_3)[name = tensor("op_442")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_482)[name = tensor("transpose_52")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_495, y = v_3)[name = tensor("inner_3")]; - tensor var_497_transpose_x_0 = const()[name = tensor("op_497_transpose_x_0"), val = tensor(false)]; - tensor var_497_transpose_y_0 = const()[name = tensor("op_497_transpose_y_0"), val = tensor(false)]; - tensor var_497 = matmul(transpose_x = var_497_transpose_x_0, transpose_y = var_497_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_497")]; - tensor var_498 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_498")]; - tensor var_499 = const()[name = tensor("op_499"), val = tensor([1, 1, 2, 1])]; - tensor var_500 = reshape(shape = var_499, x = var_498)[name = tensor("op_500")]; - tensor cross_3 = mul(x = var_497, y = var_500)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_429)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_442, y = v_3)[name = tensor("inner_3")]; + tensor var_444_transpose_x_0 = const()[name = tensor("op_444_transpose_x_0"), val = tensor(false)]; + tensor var_444_transpose_y_0 = const()[name = tensor("op_444_transpose_y_0"), val = tensor(false)]; + tensor var_444 = matmul(transpose_x = var_444_transpose_x_0, transpose_y = var_444_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_444")]; + tensor var_445 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_445")]; + tensor var_446 = const()[name = tensor("op_446"), val = tensor([1, 1, 2, 1])]; + tensor var_447 = reshape(shape = var_446, x = var_445)[name = tensor("op_447")]; + tensor cross_3 = mul(x = var_444, y = var_447)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_503 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_503")]; - tensor var_505_transpose_x_1 = const()[name = tensor("op_505_transpose_x_1"), val = tensor(true)]; - tensor var_505_transpose_y_1 = const()[name = tensor("op_505_transpose_y_1"), val = tensor(false)]; - tensor var_505 = matmul(transpose_x = var_505_transpose_x_1, transpose_y = var_505_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_505")]; - tensor new_kv_unnorm_3 = add(x = var_503, y = var_505)[name = tensor("new_kv_unnorm_3")]; - tensor var_507 = const()[name = tensor("op_507"), val = tensor(0x1p+1)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_507)[name = tensor("new_scale_3")]; - tensor var_509 = sqrt(x = new_scale_3)[name = tensor("op_509")]; - tensor var_510 = real_div(x = new_kv_unnorm_3, y = var_509)[name = tensor("op_510")]; - tensor var_511_perm_0 = const()[name = tensor("op_511_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_450 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_450")]; + tensor var_452_transpose_x_1 = const()[name = tensor("op_452_transpose_x_1"), val = tensor(true)]; + tensor var_452_transpose_y_1 = const()[name = tensor("op_452_transpose_y_1"), val = tensor(false)]; + tensor var_452 = matmul(transpose_x = var_452_transpose_x_1, transpose_y = var_452_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_452")]; + tensor new_kv_unnorm_3 = add(x = var_450, y = var_452)[name = tensor("new_kv_unnorm_3")]; + tensor var_454 = const()[name = tensor("op_454"), val = tensor(0x1p+1)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_454)[name = tensor("new_scale_3")]; + tensor var_456 = sqrt(x = new_scale_3)[name = tensor("op_456")]; + tensor var_457 = real_div(x = new_kv_unnorm_3, y = var_456)[name = tensor("op_457")]; + tensor var_458_perm_0 = const()[name = tensor("op_458_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_511 = transpose(perm = var_511_perm_0, x = out_7)[name = tensor("transpose_51")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_511)[name = tensor("out_9")]; - tensor var_515 = const()[name = tensor("op_515"), val = tensor([1, 2, 256])]; - tensor out_11 = reshape(shape = var_515, x = out_9)[name = tensor("out_11")]; - tensor var_517 = silu(x = input_67)[name = tensor("op_517")]; - tensor input_69 = mul(x = var_517, y = out_11)[name = tensor("input_69")]; - tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_69)[name = tensor("linear_16")]; - tensor x_11 = add(x = input_63, y = ret_out_3)[name = tensor("x_11")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor var_458 = transpose(perm = var_458_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_458)[name = tensor("out_9")]; + tensor var_462 = const()[name = tensor("op_462"), val = tensor([1, 2, 256])]; + tensor out_11 = reshape(shape = var_462, x = out_9)[name = tensor("out_11")]; + tensor var_464 = silu(x = input_57)[name = tensor("op_464")]; + tensor input_59 = mul(x = var_464, y = out_11)[name = tensor("input_59")]; + tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; + tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; + tensor window_7_begin_0 = const()[name = tensor("window_7_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_7_end_0 = const()[name = tensor("window_7_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_7_end_mask_0 = const()[name = tensor("window_7_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_7_squeeze_mask_0 = const()[name = tensor("window_7_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_7 = slice_by_index(begin = window_7_begin_0, end = window_7_end_0, end_mask = window_7_end_mask_0, squeeze_mask = window_7_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_7")]; tensor filled_5_begin_0 = const()[name = tensor("filled_5_begin_0"), val = tensor([1, 0])]; tensor filled_5_end_0 = const()[name = tensor("filled_5_end_0"), val = tensor([2, 1])]; tensor filled_5_end_mask_0 = const()[name = tensor("filled_5_end_mask_0"), val = tensor([false, true])]; tensor filled_5_squeeze_mask_0 = const()[name = tensor("filled_5_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_5 = slice_by_index(begin = filled_5_begin_0, end = filled_5_end_0, end_mask = filled_5_end_mask_0, squeeze_mask = filled_5_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_5")]; - tensor var_526_begin_0 = const()[name = tensor("op_526_begin_0"), val = tensor([0, 0, 0])]; - tensor var_526_end_0 = const()[name = tensor("op_526_end_0"), val = tensor([1, 1, 256])]; - tensor var_526_end_mask_0 = const()[name = tensor("op_526_end_mask_0"), val = tensor([true, false, true])]; - tensor var_526 = slice_by_index(begin = var_526_begin_0, end = var_526_end_0, end_mask = var_526_end_mask_0, x = x_11)[name = tensor("op_526")]; - tensor var_529_begin_0 = const()[name = tensor("op_529_begin_0"), val = tensor([0, 1, 0])]; - tensor var_529_end_0 = const()[name = tensor("op_529_end_0"), val = tensor([1, 16, 256])]; - tensor var_529_end_mask_0 = const()[name = tensor("op_529_end_mask_0"), val = tensor([true, true, true])]; - tensor var_529 = slice_by_index(begin = var_529_begin_0, end = var_529_end_0, end_mask = var_529_end_mask_0, x = window_3)[name = tensor("op_529")]; + tensor var_473_begin_0 = const()[name = tensor("op_473_begin_0"), val = tensor([0, 0, 0])]; + tensor var_473_end_0 = const()[name = tensor("op_473_end_0"), val = tensor([1, 1, 256])]; + tensor var_473_end_mask_0 = const()[name = tensor("op_473_end_mask_0"), val = tensor([true, false, true])]; + tensor var_473 = slice_by_index(begin = var_473_begin_0, end = var_473_end_0, end_mask = var_473_end_mask_0, x = x_9)[name = tensor("op_473")]; + tensor var_476_begin_0 = const()[name = tensor("op_476_begin_0"), val = tensor([0, 1, 0])]; + tensor var_476_end_0 = const()[name = tensor("op_476_end_0"), val = tensor([1, 16, 256])]; + tensor var_476_end_mask_0 = const()[name = tensor("op_476_end_mask_0"), val = tensor([true, true, true])]; + tensor var_476 = slice_by_index(begin = var_476_begin_0, end = var_476_end_0, end_mask = var_476_end_mask_0, x = window_7)[name = tensor("op_476")]; tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_529, var_526))[name = tensor("rolled_5")]; + tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_476, var_473))[name = tensor("rolled_5")]; tensor clip_4 = clip(alpha = var_21, beta = var_20, x = filled_5)[name = tensor("clip_4")]; - tensor var_534 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_534")]; - tensor var_535_dtype_0 = const()[name = tensor("op_535_dtype_0"), val = tensor("fp32")]; - tensor var_536 = const()[name = tensor("op_536"), val = tensor([1, 16, 1])]; - tensor var_535 = cast(dtype = var_535_dtype_0, x = var_534)[name = tensor("cast_139")]; - tensor fill_mask_5 = reshape(shape = var_536, x = var_535)[name = tensor("fill_mask_5")]; + tensor var_481 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_481")]; + tensor var_482_dtype_0 = const()[name = tensor("op_482_dtype_0"), val = tensor("fp32")]; + tensor var_483 = const()[name = tensor("op_483"), val = tensor([1, 16, 1])]; + tensor var_482 = cast(dtype = var_482_dtype_0, x = var_481)[name = tensor("cast_139")]; + tensor fill_mask_5 = reshape(shape = var_483, x = var_482)[name = tensor("fill_mask_5")]; tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_526)[name = tensor("x_expanded_5")]; - tensor var_540 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_540")]; - tensor var_541 = mul(x = window_3, y = var_540)[name = tensor("op_541")]; - tensor var_542 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_542")]; - tensor filled_window_5 = add(x = var_541, y = var_542)[name = tensor("filled_window_5")]; - tensor var_544 = greater_equal(x = filled_5, y = var_16)[name = tensor("op_544")]; - tensor var_545_dtype_0 = const()[name = tensor("op_545_dtype_0"), val = tensor("fp32")]; - tensor var_546 = const()[name = tensor("op_546"), val = tensor([1, 1, 1])]; - tensor var_545 = cast(dtype = var_545_dtype_0, x = var_544)[name = tensor("cast_138")]; - tensor use_roll_5 = reshape(shape = var_546, x = var_545)[name = tensor("use_roll_5")]; - tensor var_548 = sub(x = var_17, y = use_roll_5)[name = tensor("op_548")]; - tensor var_549 = mul(x = filled_window_5, y = var_548)[name = tensor("op_549")]; - tensor var_550 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_550")]; - tensor input_71 = add(x = var_549, y = var_550)[name = tensor("input_71")]; - tensor var_552 = const()[name = tensor("op_552"), val = tensor(0x1p+0)]; - tensor var_553 = add(x = filled_5, y = var_552)[name = tensor("op_553")]; - tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_553)[name = tensor("clip_5")]; - tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; - tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_71)[name = tensor("x_13")]; - tensor input_73_perm_0 = const()[name = tensor("input_73_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_19_pad_type_0 = const()[name = tensor("inputs_19_pad_type_0"), val = tensor("valid")]; - tensor inputs_19_strides_0 = const()[name = tensor("inputs_19_strides_0"), val = tensor([1])]; - tensor inputs_19_pad_0 = const()[name = tensor("inputs_19_pad_0"), val = tensor([0, 0])]; - tensor inputs_19_dilations_0 = const()[name = tensor("inputs_19_dilations_0"), val = tensor([1])]; - tensor inputs_19_groups_0 = const()[name = tensor("inputs_19_groups_0"), val = tensor(1)]; - tensor input_73 = transpose(perm = input_73_perm_0, x = x_13)[name = tensor("transpose_50")]; - tensor inputs_19 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_19_dilations_0, groups = inputs_19_groups_0, pad = inputs_19_pad_0, pad_type = inputs_19_pad_type_0, strides = inputs_19_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_73)[name = tensor("inputs_19")]; - tensor var_574_split_sizes_0 = const()[name = tensor("op_574_split_sizes_0"), val = tensor([256, 256])]; - tensor var_574_axis_0 = const()[name = tensor("op_574_axis_0"), val = tensor(1)]; - tensor var_574_0, tensor var_574_1 = split(axis = var_574_axis_0, split_sizes = var_574_split_sizes_0, x = inputs_19)[name = tensor("op_574")]; - tensor var_576 = sigmoid(x = var_574_1)[name = tensor("op_576")]; - tensor inputs_21 = mul(x = var_574_0, y = var_576)[name = tensor("inputs_21")]; - tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; - tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; - tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; - tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_21)[name = tensor("outputs_aug_5")]; - tensor input_75_begin_0 = const()[name = tensor("input_75_begin_0"), val = tensor([0, 0, 0])]; - tensor input_75_end_0 = const()[name = tensor("input_75_end_0"), val = tensor([1, 256, 16])]; - tensor input_75_end_mask_0 = const()[name = tensor("input_75_end_mask_0"), val = tensor([true, true, false])]; - tensor input_75 = slice_by_index(begin = input_75_begin_0, end = input_75_end_0, end_mask = input_75_end_mask_0, x = outputs_aug_5)[name = tensor("input_75")]; - tensor inputs_23 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_75)[name = tensor("inputs_23")]; - tensor input_77 = silu(x = inputs_23)[name = tensor("input_77")]; - tensor input_79_pad_type_0 = const()[name = tensor("input_79_pad_type_0"), val = tensor("valid")]; - tensor input_79_strides_0 = const()[name = tensor("input_79_strides_0"), val = tensor([1])]; - tensor input_79_pad_0 = const()[name = tensor("input_79_pad_0"), val = tensor([0, 0])]; - tensor input_79_dilations_0 = const()[name = tensor("input_79_dilations_0"), val = tensor([1])]; - tensor input_79_groups_0 = const()[name = tensor("input_79_groups_0"), val = tensor(1)]; - tensor input_79 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_79_dilations_0, groups = input_79_groups_0, pad = input_79_pad_0, pad_type = input_79_pad_type_0, strides = input_79_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_77)[name = tensor("input_79")]; - tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_607_begin_0 = const()[name = tensor("op_607_begin_0"), val = tensor([0, -1, 0])]; - tensor var_607_end_0 = const()[name = tensor("op_607_end_0"), val = tensor([1, 16, 256])]; - tensor var_607_end_mask_0 = const()[name = tensor("op_607_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_79)[name = tensor("transpose_49")]; - tensor var_607 = slice_by_index(begin = var_607_begin_0, end = var_607_end_0, end_mask = var_607_end_mask_0, x = conv_out_5)[name = tensor("op_607")]; - tensor var_610_begin_0 = const()[name = tensor("op_610_begin_0"), val = tensor([0, 1, 0])]; - tensor var_610_end_0 = const()[name = tensor("op_610_end_0"), val = tensor([1, 1, 256])]; - tensor var_610_end_mask_0 = const()[name = tensor("op_610_end_mask_0"), val = tensor([true, true, true])]; - tensor var_610 = slice_by_index(begin = var_610_begin_0, end = var_610_end_0, end_mask = var_610_end_mask_0, x = x_11)[name = tensor("op_610")]; - tensor var_613_begin_0 = const()[name = tensor("op_613_begin_0"), val = tensor([0, 1, 0])]; - tensor var_613_end_0 = const()[name = tensor("op_613_end_0"), val = tensor([1, 16, 256])]; - tensor var_613_end_mask_0 = const()[name = tensor("op_613_end_mask_0"), val = tensor([true, true, true])]; - tensor var_613 = slice_by_index(begin = var_613_begin_0, end = var_613_end_0, end_mask = var_613_end_mask_0, x = input_71)[name = tensor("op_613")]; + tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_473)[name = tensor("x_expanded_5")]; + tensor var_487 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_487")]; + tensor var_488 = mul(x = window_7, y = var_487)[name = tensor("op_488")]; + tensor var_489 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_489")]; + tensor filled_window_5 = add(x = var_488, y = var_489)[name = tensor("filled_window_5")]; + tensor var_491 = greater_equal(x = filled_5, y = var_16)[name = tensor("op_491")]; + tensor var_492_dtype_0 = const()[name = tensor("op_492_dtype_0"), val = tensor("fp32")]; + tensor var_493 = const()[name = tensor("op_493"), val = tensor([1, 1, 1])]; + tensor var_492 = cast(dtype = var_492_dtype_0, x = var_491)[name = tensor("cast_138")]; + tensor use_roll_5 = reshape(shape = var_493, x = var_492)[name = tensor("use_roll_5")]; + tensor var_495 = sub(x = var_17, y = use_roll_5)[name = tensor("op_495")]; + tensor var_496 = mul(x = filled_window_5, y = var_495)[name = tensor("op_496")]; + tensor var_497 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_497")]; + tensor window_9 = add(x = var_496, y = var_497)[name = tensor("window_9")]; + tensor var_499 = const()[name = tensor("op_499"), val = tensor(0x1p+0)]; + tensor var_500 = add(x = filled_5, y = var_499)[name = tensor("op_500")]; + tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_500)[name = tensor("clip_5")]; + tensor var_503_begin_0 = const()[name = tensor("op_503_begin_0"), val = tensor([0, 1, 0])]; + tensor var_503_end_0 = const()[name = tensor("op_503_end_0"), val = tensor([1, 1, 256])]; + tensor var_503_end_mask_0 = const()[name = tensor("op_503_end_mask_0"), val = tensor([true, true, true])]; + tensor var_503 = slice_by_index(begin = var_503_begin_0, end = var_503_end_0, end_mask = var_503_end_mask_0, x = x_9)[name = tensor("op_503")]; + tensor var_506_begin_0 = const()[name = tensor("op_506_begin_0"), val = tensor([0, 1, 0])]; + tensor var_506_end_0 = const()[name = tensor("op_506_end_0"), val = tensor([1, 16, 256])]; + tensor var_506_end_mask_0 = const()[name = tensor("op_506_end_mask_0"), val = tensor([true, true, true])]; + tensor var_506 = slice_by_index(begin = var_506_begin_0, end = var_506_end_0, end_mask = var_506_end_mask_0, x = window_9)[name = tensor("op_506")]; tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; - tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_613, var_610))[name = tensor("rolled_7")]; + tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_506, var_503))[name = tensor("rolled_7")]; tensor clip_6 = clip(alpha = var_21, beta = var_20, x = clip_5)[name = tensor("clip_6")]; - tensor var_618 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_618")]; - tensor var_619_dtype_0 = const()[name = tensor("op_619_dtype_0"), val = tensor("fp32")]; - tensor var_620 = const()[name = tensor("op_620"), val = tensor([1, 16, 1])]; - tensor var_619 = cast(dtype = var_619_dtype_0, x = var_618)[name = tensor("cast_137")]; - tensor fill_mask_7 = reshape(shape = var_620, x = var_619)[name = tensor("fill_mask_7")]; + tensor var_511 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_511")]; + tensor var_512_dtype_0 = const()[name = tensor("op_512_dtype_0"), val = tensor("fp32")]; + tensor var_513 = const()[name = tensor("op_513"), val = tensor([1, 16, 1])]; + tensor var_512 = cast(dtype = var_512_dtype_0, x = var_511)[name = tensor("cast_137")]; + tensor fill_mask_7 = reshape(shape = var_513, x = var_512)[name = tensor("fill_mask_7")]; tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_610)[name = tensor("x_expanded_7")]; - tensor var_624 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_624")]; - tensor var_625 = mul(x = input_71, y = var_624)[name = tensor("op_625")]; - tensor var_626 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_626")]; - tensor filled_window_7 = add(x = var_625, y = var_626)[name = tensor("filled_window_7")]; - tensor var_628 = greater_equal(x = clip_5, y = var_16)[name = tensor("op_628")]; - tensor var_629_dtype_0 = const()[name = tensor("op_629_dtype_0"), val = tensor("fp32")]; - tensor var_630 = const()[name = tensor("op_630"), val = tensor([1, 1, 1])]; - tensor var_629 = cast(dtype = var_629_dtype_0, x = var_628)[name = tensor("cast_136")]; - tensor use_roll_7 = reshape(shape = var_630, x = var_629)[name = tensor("use_roll_7")]; - tensor var_632 = sub(x = var_17, y = use_roll_7)[name = tensor("op_632")]; - tensor var_633 = mul(x = filled_window_7, y = var_632)[name = tensor("op_633")]; - tensor var_634 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_634")]; - tensor input_81 = add(x = var_633, y = var_634)[name = tensor("input_81")]; - tensor var_636 = const()[name = tensor("op_636"), val = tensor(0x1p+0)]; - tensor var_637 = add(x = clip_5, y = var_636)[name = tensor("op_637")]; - tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_637)[name = tensor("clip_7")]; - tensor x_15_axes_0 = const()[name = tensor("x_15_axes_0"), val = tensor([-1])]; - tensor x_15 = layer_norm(axes = x_15_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_81)[name = tensor("x_15")]; - tensor input_83_perm_0 = const()[name = tensor("input_83_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_25_pad_type_0 = const()[name = tensor("inputs_25_pad_type_0"), val = tensor("valid")]; - tensor inputs_25_strides_0 = const()[name = tensor("inputs_25_strides_0"), val = tensor([1])]; - tensor inputs_25_pad_0 = const()[name = tensor("inputs_25_pad_0"), val = tensor([0, 0])]; - tensor inputs_25_dilations_0 = const()[name = tensor("inputs_25_dilations_0"), val = tensor([1])]; - tensor inputs_25_groups_0 = const()[name = tensor("inputs_25_groups_0"), val = tensor(1)]; - tensor input_83 = transpose(perm = input_83_perm_0, x = x_15)[name = tensor("transpose_48")]; - tensor inputs_25 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_25_dilations_0, groups = inputs_25_groups_0, pad = inputs_25_pad_0, pad_type = inputs_25_pad_type_0, strides = inputs_25_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_83)[name = tensor("inputs_25")]; - tensor var_658_split_sizes_0 = const()[name = tensor("op_658_split_sizes_0"), val = tensor([256, 256])]; - tensor var_658_axis_0 = const()[name = tensor("op_658_axis_0"), val = tensor(1)]; - tensor var_658_0, tensor var_658_1 = split(axis = var_658_axis_0, split_sizes = var_658_split_sizes_0, x = inputs_25)[name = tensor("op_658")]; - tensor var_660 = sigmoid(x = var_658_1)[name = tensor("op_660")]; - tensor inputs_27 = mul(x = var_658_0, y = var_660)[name = tensor("inputs_27")]; - tensor outputs_aug_7_pad_type_0 = const()[name = tensor("outputs_aug_7_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_7_pad_0 = const()[name = tensor("outputs_aug_7_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_7_groups_0 = const()[name = tensor("outputs_aug_7_groups_0"), val = tensor(256)]; - tensor outputs_aug_7_strides_0 = const()[name = tensor("outputs_aug_7_strides_0"), val = tensor([1])]; - tensor outputs_aug_7_dilations_0 = const()[name = tensor("outputs_aug_7_dilations_0"), val = tensor([1])]; - tensor outputs_aug_7 = conv(dilations = outputs_aug_7_dilations_0, groups = outputs_aug_7_groups_0, pad = outputs_aug_7_pad_0, pad_type = outputs_aug_7_pad_type_0, strides = outputs_aug_7_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_27)[name = tensor("outputs_aug_7")]; - tensor input_85_begin_0 = const()[name = tensor("input_85_begin_0"), val = tensor([0, 0, 0])]; - tensor input_85_end_0 = const()[name = tensor("input_85_end_0"), val = tensor([1, 256, 16])]; - tensor input_85_end_mask_0 = const()[name = tensor("input_85_end_mask_0"), val = tensor([true, true, false])]; - tensor input_85 = slice_by_index(begin = input_85_begin_0, end = input_85_end_0, end_mask = input_85_end_mask_0, x = outputs_aug_7)[name = tensor("input_85")]; - tensor inputs_29 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_85)[name = tensor("inputs_29")]; - tensor input_87 = silu(x = inputs_29)[name = tensor("input_87")]; - tensor input_89_pad_type_0 = const()[name = tensor("input_89_pad_type_0"), val = tensor("valid")]; - tensor input_89_strides_0 = const()[name = tensor("input_89_strides_0"), val = tensor([1])]; - tensor input_89_pad_0 = const()[name = tensor("input_89_pad_0"), val = tensor([0, 0])]; - tensor input_89_dilations_0 = const()[name = tensor("input_89_dilations_0"), val = tensor([1])]; - tensor input_89_groups_0 = const()[name = tensor("input_89_groups_0"), val = tensor(1)]; - tensor input_89 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_89_dilations_0, groups = input_89_groups_0, pad = input_89_pad_0, pad_type = input_89_pad_type_0, strides = input_89_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_87)[name = tensor("input_89")]; - tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_691_begin_0 = const()[name = tensor("op_691_begin_0"), val = tensor([0, -1, 0])]; - tensor var_691_end_0 = const()[name = tensor("op_691_end_0"), val = tensor([1, 16, 256])]; - tensor var_691_end_mask_0 = const()[name = tensor("op_691_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_89)[name = tensor("transpose_47")]; - tensor var_691 = slice_by_index(begin = var_691_begin_0, end = var_691_end_0, end_mask = var_691_end_mask_0, x = conv_out_7)[name = tensor("op_691")]; - tensor var_694_interleave_0 = const()[name = tensor("op_694_interleave_0"), val = tensor(false)]; - tensor var_694 = concat(axis = var_33, interleave = var_694_interleave_0, values = (var_607, var_691))[name = tensor("op_694")]; - tensor input_91 = add(x = x_11, y = var_694)[name = tensor("input_91")]; - tensor input_93_axes_0 = const()[name = tensor("input_93_axes_0"), val = tensor([-1])]; - tensor input_93 = layer_norm(axes = input_93_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_91)[name = tensor("input_93")]; - tensor inputs_31 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_93)[name = tensor("linear_17")]; - tensor input_95 = silu(x = inputs_31)[name = tensor("input_95")]; - tensor input_99 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_95)[name = tensor("linear_18")]; - tensor var_717 = const()[name = tensor("op_717"), val = tensor(0x1p-1)]; - tensor var_718 = mul(x = input_99, y = var_717)[name = tensor("op_718")]; - tensor input_101 = add(x = var_718, y = input_91)[name = tensor("input_101")]; - tensor input_103_axes_0 = const()[name = tensor("input_103_axes_0"), val = tensor([-1])]; - tensor input_103 = layer_norm(axes = input_103_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_101)[name = tensor("input_103")]; - tensor input_105_axes_0 = const()[name = tensor("input_105_axes_0"), val = tensor([-1])]; - tensor input_105 = layer_norm(axes = input_105_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_103)[name = tensor("input_105")]; - tensor inputs_33 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_105)[name = tensor("linear_19")]; - tensor input_107 = silu(x = inputs_33)[name = tensor("input_107")]; - tensor input_111 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_107)[name = tensor("linear_20")]; - tensor var_747 = const()[name = tensor("op_747"), val = tensor(0x1p-1)]; - tensor var_748 = mul(x = input_111, y = var_747)[name = tensor("op_748")]; - tensor input_113 = add(x = var_748, y = input_103)[name = tensor("input_113")]; - tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; - tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_113)[name = tensor("x_17")]; + tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_503)[name = tensor("x_expanded_7")]; + tensor var_517 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_517")]; + tensor var_518 = mul(x = window_9, y = var_517)[name = tensor("op_518")]; + tensor var_519 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_519")]; + tensor filled_window_7 = add(x = var_518, y = var_519)[name = tensor("filled_window_7")]; + tensor var_521 = greater_equal(x = clip_5, y = var_16)[name = tensor("op_521")]; + tensor var_522_dtype_0 = const()[name = tensor("op_522_dtype_0"), val = tensor("fp32")]; + tensor var_523 = const()[name = tensor("op_523"), val = tensor([1, 1, 1])]; + tensor var_522 = cast(dtype = var_522_dtype_0, x = var_521)[name = tensor("cast_136")]; + tensor use_roll_7 = reshape(shape = var_523, x = var_522)[name = tensor("use_roll_7")]; + tensor var_525 = sub(x = var_17, y = use_roll_7)[name = tensor("op_525")]; + tensor var_526 = mul(x = filled_window_7, y = var_525)[name = tensor("op_526")]; + tensor var_527 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_527")]; + tensor window_11 = add(x = var_526, y = var_527)[name = tensor("window_11")]; + tensor var_529 = const()[name = tensor("op_529"), val = tensor(0x1p+0)]; + tensor var_530 = add(x = clip_5, y = var_529)[name = tensor("op_530")]; + tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_530)[name = tensor("clip_7")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_30, interleave = input_61_interleave_0, values = (window_9, window_11))[name = tensor("input_61")]; + tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; + tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; + tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_13_pad_type_0 = const()[name = tensor("inputs_13_pad_type_0"), val = tensor("valid")]; + tensor inputs_13_strides_0 = const()[name = tensor("inputs_13_strides_0"), val = tensor([1])]; + tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; + tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; + tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; + tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; + tensor var_553_split_sizes_0 = const()[name = tensor("op_553_split_sizes_0"), val = tensor([256, 256])]; + tensor var_553_axis_0 = const()[name = tensor("op_553_axis_0"), val = tensor(1)]; + tensor var_553_0, tensor var_553_1 = split(axis = var_553_axis_0, split_sizes = var_553_split_sizes_0, x = inputs_13)[name = tensor("op_553")]; + tensor var_555 = sigmoid(x = var_553_1)[name = tensor("op_555")]; + tensor inputs_15 = mul(x = var_553_0, y = var_555)[name = tensor("inputs_15")]; + tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; + tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; + tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; + tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_15)[name = tensor("outputs_aug_3")]; + tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; + tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([2, 256, 16])]; + tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; + tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_3)[name = tensor("input_65")]; + tensor inputs_17 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_65)[name = tensor("inputs_17")]; + tensor input_67 = silu(x = inputs_17)[name = tensor("input_67")]; + tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; + tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; + tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; + tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; + tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; + tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; + tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; + tensor var_586_begin_0 = const()[name = tensor("op_586_begin_0"), val = tensor([0, -1, 0])]; + tensor var_586_end_0 = const()[name = tensor("op_586_end_0"), val = tensor([2, 16, 256])]; + tensor var_586_end_mask_0 = const()[name = tensor("op_586_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_586 = slice_by_index(begin = var_586_begin_0, end = var_586_end_0, end_mask = var_586_end_mask_0, x = conv_out_3)[name = tensor("op_586")]; + tensor var_588_perm_0 = const()[name = tensor("op_588_perm_0"), val = tensor([1, 0, 2])]; + tensor var_588 = transpose(perm = var_588_perm_0, x = var_586)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_588)[name = tensor("input_71")]; + tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; + tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; + tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; + tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; + tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; + tensor var_611 = const()[name = tensor("op_611"), val = tensor(0x1p-1)]; + tensor var_612 = mul(x = input_79, y = var_611)[name = tensor("op_612")]; + tensor input_81 = add(x = var_612, y = input_71)[name = tensor("input_81")]; + tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; + tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; + tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; + tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; + tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; + tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; + tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; + tensor var_641 = const()[name = tensor("op_641"), val = tensor(0x1p-1)]; + tensor var_642 = mul(x = input_91, y = var_641)[name = tensor("op_642")]; + tensor input_93 = add(x = var_642, y = input_83)[name = tensor("input_93")]; + tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; + tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; tensor prev_kv_5_end_0 = const()[name = tensor("prev_kv_5_end_0"), val = tensor([3, 1, 4, 64, 64])]; tensor prev_kv_5_end_mask_0 = const()[name = tensor("prev_kv_5_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -737,243 +664,206 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_762 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_17)[name = tensor("linear_21")]; - tensor var_763 = const()[name = tensor("op_763"), val = tensor([1, 2, 4, 64])]; - tensor var_764 = reshape(shape = var_763, x = var_762)[name = tensor("op_764")]; + tensor var_656 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_657 = const()[name = tensor("op_657"), val = tensor([1, 2, 4, 64])]; + tensor var_658 = reshape(shape = var_657, x = var_656)[name = tensor("op_658")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_768 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_17)[name = tensor("linear_22")]; - tensor var_769 = const()[name = tensor("op_769"), val = tensor(0x1p-3)]; - tensor var_770 = mul(x = var_768, y = var_769)[name = tensor("op_770")]; - tensor var_771 = const()[name = tensor("op_771"), val = tensor([1, 2, 4, 64])]; - tensor var_772 = reshape(shape = var_771, x = var_770)[name = tensor("op_772")]; + tensor var_662 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_663 = const()[name = tensor("op_663"), val = tensor(0x1p-3)]; + tensor var_664 = mul(x = var_662, y = var_663)[name = tensor("op_664")]; + tensor var_665 = const()[name = tensor("op_665"), val = tensor([1, 2, 4, 64])]; + tensor var_666 = reshape(shape = var_665, x = var_664)[name = tensor("op_666")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_776 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_17)[name = tensor("linear_23")]; - tensor var_777 = const()[name = tensor("op_777"), val = tensor([1, 2, 4, 64])]; - tensor var_778 = reshape(shape = var_777, x = var_776)[name = tensor("op_778")]; + tensor var_670 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_671 = const()[name = tensor("op_671"), val = tensor([1, 2, 4, 64])]; + tensor var_672 = reshape(shape = var_671, x = var_670)[name = tensor("op_672")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_117 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_17)[name = tensor("linear_24")]; + tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; tensor s_t_5 = add(x = prev_scale_5, y = encoder__t_index)[name = tensor("s_t_5")]; tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_772)[name = tensor("transpose_45")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_764)[name = tensor("transpose_46")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_666)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_658)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_788 = const()[name = tensor("op_788"), val = tensor([2, 1])]; - tensor var_789 = reshape(shape = var_788, x = sqrt_s_t_5)[name = tensor("op_789")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_789)[name = tensor("M_5")]; - tensor var_791 = mul(x = qk_5, y = M_5)[name = tensor("op_791")]; + tensor var_682 = const()[name = tensor("op_682"), val = tensor([2, 1])]; + tensor var_683 = reshape(shape = var_682, x = sqrt_s_t_5)[name = tensor("op_683")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_683)[name = tensor("M_5")]; + tensor var_685 = mul(x = qk_5, y = M_5)[name = tensor("op_685")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_778)[name = tensor("transpose_44")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_791, y = v_5)[name = tensor("inner_5")]; - tensor var_793_transpose_x_0 = const()[name = tensor("op_793_transpose_x_0"), val = tensor(false)]; - tensor var_793_transpose_y_0 = const()[name = tensor("op_793_transpose_y_0"), val = tensor(false)]; - tensor var_793 = matmul(transpose_x = var_793_transpose_x_0, transpose_y = var_793_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_793")]; - tensor var_794 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_794")]; - tensor var_795 = const()[name = tensor("op_795"), val = tensor([1, 1, 2, 1])]; - tensor var_796 = reshape(shape = var_795, x = var_794)[name = tensor("op_796")]; - tensor cross_5 = mul(x = var_793, y = var_796)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_672)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_685, y = v_5)[name = tensor("inner_5")]; + tensor var_687_transpose_x_0 = const()[name = tensor("op_687_transpose_x_0"), val = tensor(false)]; + tensor var_687_transpose_y_0 = const()[name = tensor("op_687_transpose_y_0"), val = tensor(false)]; + tensor var_687 = matmul(transpose_x = var_687_transpose_x_0, transpose_y = var_687_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_687")]; + tensor var_688 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_688")]; + tensor var_689 = const()[name = tensor("op_689"), val = tensor([1, 1, 2, 1])]; + tensor var_690 = reshape(shape = var_689, x = var_688)[name = tensor("op_690")]; + tensor cross_5 = mul(x = var_687, y = var_690)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_799 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_799")]; - tensor var_801_transpose_x_1 = const()[name = tensor("op_801_transpose_x_1"), val = tensor(true)]; - tensor var_801_transpose_y_1 = const()[name = tensor("op_801_transpose_y_1"), val = tensor(false)]; - tensor var_801 = matmul(transpose_x = var_801_transpose_x_1, transpose_y = var_801_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_801")]; - tensor new_kv_unnorm_5 = add(x = var_799, y = var_801)[name = tensor("new_kv_unnorm_5")]; - tensor var_803 = const()[name = tensor("op_803"), val = tensor(0x1p+1)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_803)[name = tensor("new_scale_5")]; - tensor var_805 = sqrt(x = new_scale_5)[name = tensor("op_805")]; - tensor var_806 = real_div(x = new_kv_unnorm_5, y = var_805)[name = tensor("op_806")]; - tensor var_807_perm_0 = const()[name = tensor("op_807_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_693 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_693")]; + tensor var_695_transpose_x_1 = const()[name = tensor("op_695_transpose_x_1"), val = tensor(true)]; + tensor var_695_transpose_y_1 = const()[name = tensor("op_695_transpose_y_1"), val = tensor(false)]; + tensor var_695 = matmul(transpose_x = var_695_transpose_x_1, transpose_y = var_695_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_695")]; + tensor new_kv_unnorm_5 = add(x = var_693, y = var_695)[name = tensor("new_kv_unnorm_5")]; + tensor var_697 = const()[name = tensor("op_697"), val = tensor(0x1p+1)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_697)[name = tensor("new_scale_5")]; + tensor var_699 = sqrt(x = new_scale_5)[name = tensor("op_699")]; + tensor var_700 = real_div(x = new_kv_unnorm_5, y = var_699)[name = tensor("op_700")]; + tensor var_701_perm_0 = const()[name = tensor("op_701_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_807 = transpose(perm = var_807_perm_0, x = out_13)[name = tensor("transpose_43")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_807)[name = tensor("out_15")]; - tensor var_811 = const()[name = tensor("op_811"), val = tensor([1, 2, 256])]; - tensor out_17 = reshape(shape = var_811, x = out_15)[name = tensor("out_17")]; - tensor var_813 = silu(x = input_117)[name = tensor("op_813")]; - tensor input_119 = mul(x = var_813, y = out_17)[name = tensor("input_119")]; - tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_119)[name = tensor("linear_25")]; - tensor x_19 = add(x = input_113, y = ret_out_5)[name = tensor("x_19")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor var_701 = transpose(perm = var_701_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_701)[name = tensor("out_15")]; + tensor var_705 = const()[name = tensor("op_705"), val = tensor([1, 2, 256])]; + tensor out_17 = reshape(shape = var_705, x = out_15)[name = tensor("out_17")]; + tensor var_707 = silu(x = input_97)[name = tensor("op_707")]; + tensor input_99 = mul(x = var_707, y = out_17)[name = tensor("input_99")]; + tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; + tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; + tensor window_13_begin_0 = const()[name = tensor("window_13_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_13_end_0 = const()[name = tensor("window_13_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_13_end_mask_0 = const()[name = tensor("window_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_13_squeeze_mask_0 = const()[name = tensor("window_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_13 = slice_by_index(begin = window_13_begin_0, end = window_13_end_0, end_mask = window_13_end_mask_0, squeeze_mask = window_13_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_13")]; tensor filled_9_begin_0 = const()[name = tensor("filled_9_begin_0"), val = tensor([2, 0])]; tensor filled_9_end_0 = const()[name = tensor("filled_9_end_0"), val = tensor([3, 1])]; tensor filled_9_end_mask_0 = const()[name = tensor("filled_9_end_mask_0"), val = tensor([false, true])]; tensor filled_9_squeeze_mask_0 = const()[name = tensor("filled_9_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_9 = slice_by_index(begin = filled_9_begin_0, end = filled_9_end_0, end_mask = filled_9_end_mask_0, squeeze_mask = filled_9_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_9")]; - tensor var_822_begin_0 = const()[name = tensor("op_822_begin_0"), val = tensor([0, 0, 0])]; - tensor var_822_end_0 = const()[name = tensor("op_822_end_0"), val = tensor([1, 1, 256])]; - tensor var_822_end_mask_0 = const()[name = tensor("op_822_end_mask_0"), val = tensor([true, false, true])]; - tensor var_822 = slice_by_index(begin = var_822_begin_0, end = var_822_end_0, end_mask = var_822_end_mask_0, x = x_19)[name = tensor("op_822")]; - tensor var_825_begin_0 = const()[name = tensor("op_825_begin_0"), val = tensor([0, 1, 0])]; - tensor var_825_end_0 = const()[name = tensor("op_825_end_0"), val = tensor([1, 16, 256])]; - tensor var_825_end_mask_0 = const()[name = tensor("op_825_end_mask_0"), val = tensor([true, true, true])]; - tensor var_825 = slice_by_index(begin = var_825_begin_0, end = var_825_end_0, end_mask = var_825_end_mask_0, x = window_5)[name = tensor("op_825")]; + tensor var_716_begin_0 = const()[name = tensor("op_716_begin_0"), val = tensor([0, 0, 0])]; + tensor var_716_end_0 = const()[name = tensor("op_716_end_0"), val = tensor([1, 1, 256])]; + tensor var_716_end_mask_0 = const()[name = tensor("op_716_end_mask_0"), val = tensor([true, false, true])]; + tensor var_716 = slice_by_index(begin = var_716_begin_0, end = var_716_end_0, end_mask = var_716_end_mask_0, x = x_15)[name = tensor("op_716")]; + tensor var_719_begin_0 = const()[name = tensor("op_719_begin_0"), val = tensor([0, 1, 0])]; + tensor var_719_end_0 = const()[name = tensor("op_719_end_0"), val = tensor([1, 16, 256])]; + tensor var_719_end_mask_0 = const()[name = tensor("op_719_end_mask_0"), val = tensor([true, true, true])]; + tensor var_719 = slice_by_index(begin = var_719_begin_0, end = var_719_end_0, end_mask = var_719_end_mask_0, x = window_13)[name = tensor("op_719")]; tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; - tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_825, var_822))[name = tensor("rolled_9")]; + tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_719, var_716))[name = tensor("rolled_9")]; tensor clip_8 = clip(alpha = var_21, beta = var_20, x = filled_9)[name = tensor("clip_8")]; - tensor var_830 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_830")]; - tensor var_831_dtype_0 = const()[name = tensor("op_831_dtype_0"), val = tensor("fp32")]; - tensor var_832 = const()[name = tensor("op_832"), val = tensor([1, 16, 1])]; - tensor var_831 = cast(dtype = var_831_dtype_0, x = var_830)[name = tensor("cast_135")]; - tensor fill_mask_9 = reshape(shape = var_832, x = var_831)[name = tensor("fill_mask_9")]; + tensor var_724 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_724")]; + tensor var_725_dtype_0 = const()[name = tensor("op_725_dtype_0"), val = tensor("fp32")]; + tensor var_726 = const()[name = tensor("op_726"), val = tensor([1, 16, 1])]; + tensor var_725 = cast(dtype = var_725_dtype_0, x = var_724)[name = tensor("cast_135")]; + tensor fill_mask_9 = reshape(shape = var_726, x = var_725)[name = tensor("fill_mask_9")]; tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_822)[name = tensor("x_expanded_9")]; - tensor var_836 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_836")]; - tensor var_837 = mul(x = window_5, y = var_836)[name = tensor("op_837")]; - tensor var_838 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_838")]; - tensor filled_window_9 = add(x = var_837, y = var_838)[name = tensor("filled_window_9")]; - tensor var_840 = greater_equal(x = filled_9, y = var_16)[name = tensor("op_840")]; - tensor var_841_dtype_0 = const()[name = tensor("op_841_dtype_0"), val = tensor("fp32")]; - tensor var_842 = const()[name = tensor("op_842"), val = tensor([1, 1, 1])]; - tensor var_841 = cast(dtype = var_841_dtype_0, x = var_840)[name = tensor("cast_134")]; - tensor use_roll_9 = reshape(shape = var_842, x = var_841)[name = tensor("use_roll_9")]; - tensor var_844 = sub(x = var_17, y = use_roll_9)[name = tensor("op_844")]; - tensor var_845 = mul(x = filled_window_9, y = var_844)[name = tensor("op_845")]; - tensor var_846 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_846")]; - tensor input_121 = add(x = var_845, y = var_846)[name = tensor("input_121")]; - tensor var_848 = const()[name = tensor("op_848"), val = tensor(0x1p+0)]; - tensor var_849 = add(x = filled_9, y = var_848)[name = tensor("op_849")]; - tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_849)[name = tensor("clip_9")]; - tensor x_21_axes_0 = const()[name = tensor("x_21_axes_0"), val = tensor([-1])]; - tensor x_21 = layer_norm(axes = x_21_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_121)[name = tensor("x_21")]; - tensor input_123_perm_0 = const()[name = tensor("input_123_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_35_pad_type_0 = const()[name = tensor("inputs_35_pad_type_0"), val = tensor("valid")]; - tensor inputs_35_strides_0 = const()[name = tensor("inputs_35_strides_0"), val = tensor([1])]; - tensor inputs_35_pad_0 = const()[name = tensor("inputs_35_pad_0"), val = tensor([0, 0])]; - tensor inputs_35_dilations_0 = const()[name = tensor("inputs_35_dilations_0"), val = tensor([1])]; - tensor inputs_35_groups_0 = const()[name = tensor("inputs_35_groups_0"), val = tensor(1)]; - tensor input_123 = transpose(perm = input_123_perm_0, x = x_21)[name = tensor("transpose_42")]; - tensor inputs_35 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_35_dilations_0, groups = inputs_35_groups_0, pad = inputs_35_pad_0, pad_type = inputs_35_pad_type_0, strides = inputs_35_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_123)[name = tensor("inputs_35")]; - tensor var_870_split_sizes_0 = const()[name = tensor("op_870_split_sizes_0"), val = tensor([256, 256])]; - tensor var_870_axis_0 = const()[name = tensor("op_870_axis_0"), val = tensor(1)]; - tensor var_870_0, tensor var_870_1 = split(axis = var_870_axis_0, split_sizes = var_870_split_sizes_0, x = inputs_35)[name = tensor("op_870")]; - tensor var_872 = sigmoid(x = var_870_1)[name = tensor("op_872")]; - tensor inputs_37 = mul(x = var_870_0, y = var_872)[name = tensor("inputs_37")]; - tensor outputs_aug_9_pad_type_0 = const()[name = tensor("outputs_aug_9_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_9_pad_0 = const()[name = tensor("outputs_aug_9_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_9_groups_0 = const()[name = tensor("outputs_aug_9_groups_0"), val = tensor(256)]; - tensor outputs_aug_9_strides_0 = const()[name = tensor("outputs_aug_9_strides_0"), val = tensor([1])]; - tensor outputs_aug_9_dilations_0 = const()[name = tensor("outputs_aug_9_dilations_0"), val = tensor([1])]; - tensor outputs_aug_9 = conv(dilations = outputs_aug_9_dilations_0, groups = outputs_aug_9_groups_0, pad = outputs_aug_9_pad_0, pad_type = outputs_aug_9_pad_type_0, strides = outputs_aug_9_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_37)[name = tensor("outputs_aug_9")]; - tensor input_125_begin_0 = const()[name = tensor("input_125_begin_0"), val = tensor([0, 0, 0])]; - tensor input_125_end_0 = const()[name = tensor("input_125_end_0"), val = tensor([1, 256, 16])]; - tensor input_125_end_mask_0 = const()[name = tensor("input_125_end_mask_0"), val = tensor([true, true, false])]; - tensor input_125 = slice_by_index(begin = input_125_begin_0, end = input_125_end_0, end_mask = input_125_end_mask_0, x = outputs_aug_9)[name = tensor("input_125")]; - tensor inputs_39 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_125)[name = tensor("inputs_39")]; - tensor input_127 = silu(x = inputs_39)[name = tensor("input_127")]; - tensor input_129_pad_type_0 = const()[name = tensor("input_129_pad_type_0"), val = tensor("valid")]; - tensor input_129_strides_0 = const()[name = tensor("input_129_strides_0"), val = tensor([1])]; - tensor input_129_pad_0 = const()[name = tensor("input_129_pad_0"), val = tensor([0, 0])]; - tensor input_129_dilations_0 = const()[name = tensor("input_129_dilations_0"), val = tensor([1])]; - tensor input_129_groups_0 = const()[name = tensor("input_129_groups_0"), val = tensor(1)]; - tensor input_129 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_129_dilations_0, groups = input_129_groups_0, pad = input_129_pad_0, pad_type = input_129_pad_type_0, strides = input_129_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_127)[name = tensor("input_129")]; - tensor conv_out_9_perm_0 = const()[name = tensor("conv_out_9_perm_0"), val = tensor([0, 2, 1])]; - tensor var_903_begin_0 = const()[name = tensor("op_903_begin_0"), val = tensor([0, -1, 0])]; - tensor var_903_end_0 = const()[name = tensor("op_903_end_0"), val = tensor([1, 16, 256])]; - tensor var_903_end_mask_0 = const()[name = tensor("op_903_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_9 = transpose(perm = conv_out_9_perm_0, x = input_129)[name = tensor("transpose_41")]; - tensor var_903 = slice_by_index(begin = var_903_begin_0, end = var_903_end_0, end_mask = var_903_end_mask_0, x = conv_out_9)[name = tensor("op_903")]; - tensor var_906_begin_0 = const()[name = tensor("op_906_begin_0"), val = tensor([0, 1, 0])]; - tensor var_906_end_0 = const()[name = tensor("op_906_end_0"), val = tensor([1, 1, 256])]; - tensor var_906_end_mask_0 = const()[name = tensor("op_906_end_mask_0"), val = tensor([true, true, true])]; - tensor var_906 = slice_by_index(begin = var_906_begin_0, end = var_906_end_0, end_mask = var_906_end_mask_0, x = x_19)[name = tensor("op_906")]; - tensor var_909_begin_0 = const()[name = tensor("op_909_begin_0"), val = tensor([0, 1, 0])]; - tensor var_909_end_0 = const()[name = tensor("op_909_end_0"), val = tensor([1, 16, 256])]; - tensor var_909_end_mask_0 = const()[name = tensor("op_909_end_mask_0"), val = tensor([true, true, true])]; - tensor var_909 = slice_by_index(begin = var_909_begin_0, end = var_909_end_0, end_mask = var_909_end_mask_0, x = input_121)[name = tensor("op_909")]; + tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_716)[name = tensor("x_expanded_9")]; + tensor var_730 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_730")]; + tensor var_731 = mul(x = window_13, y = var_730)[name = tensor("op_731")]; + tensor var_732 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_732")]; + tensor filled_window_9 = add(x = var_731, y = var_732)[name = tensor("filled_window_9")]; + tensor var_734 = greater_equal(x = filled_9, y = var_16)[name = tensor("op_734")]; + tensor var_735_dtype_0 = const()[name = tensor("op_735_dtype_0"), val = tensor("fp32")]; + tensor var_736 = const()[name = tensor("op_736"), val = tensor([1, 1, 1])]; + tensor var_735 = cast(dtype = var_735_dtype_0, x = var_734)[name = tensor("cast_134")]; + tensor use_roll_9 = reshape(shape = var_736, x = var_735)[name = tensor("use_roll_9")]; + tensor var_738 = sub(x = var_17, y = use_roll_9)[name = tensor("op_738")]; + tensor var_739 = mul(x = filled_window_9, y = var_738)[name = tensor("op_739")]; + tensor var_740 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_740")]; + tensor window_15 = add(x = var_739, y = var_740)[name = tensor("window_15")]; + tensor var_742 = const()[name = tensor("op_742"), val = tensor(0x1p+0)]; + tensor var_743 = add(x = filled_9, y = var_742)[name = tensor("op_743")]; + tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_743)[name = tensor("clip_9")]; + tensor var_746_begin_0 = const()[name = tensor("op_746_begin_0"), val = tensor([0, 1, 0])]; + tensor var_746_end_0 = const()[name = tensor("op_746_end_0"), val = tensor([1, 1, 256])]; + tensor var_746_end_mask_0 = const()[name = tensor("op_746_end_mask_0"), val = tensor([true, true, true])]; + tensor var_746 = slice_by_index(begin = var_746_begin_0, end = var_746_end_0, end_mask = var_746_end_mask_0, x = x_15)[name = tensor("op_746")]; + tensor var_749_begin_0 = const()[name = tensor("op_749_begin_0"), val = tensor([0, 1, 0])]; + tensor var_749_end_0 = const()[name = tensor("op_749_end_0"), val = tensor([1, 16, 256])]; + tensor var_749_end_mask_0 = const()[name = tensor("op_749_end_mask_0"), val = tensor([true, true, true])]; + tensor var_749 = slice_by_index(begin = var_749_begin_0, end = var_749_end_0, end_mask = var_749_end_mask_0, x = window_15)[name = tensor("op_749")]; tensor rolled_11_interleave_0 = const()[name = tensor("rolled_11_interleave_0"), val = tensor(false)]; - tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_909, var_906))[name = tensor("rolled_11")]; + tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_749, var_746))[name = tensor("rolled_11")]; tensor clip_10 = clip(alpha = var_21, beta = var_20, x = clip_9)[name = tensor("clip_10")]; - tensor var_914 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_914")]; - tensor var_915_dtype_0 = const()[name = tensor("op_915_dtype_0"), val = tensor("fp32")]; - tensor var_916 = const()[name = tensor("op_916"), val = tensor([1, 16, 1])]; - tensor var_915 = cast(dtype = var_915_dtype_0, x = var_914)[name = tensor("cast_133")]; - tensor fill_mask_11 = reshape(shape = var_916, x = var_915)[name = tensor("fill_mask_11")]; + tensor var_754 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_754")]; + tensor var_755_dtype_0 = const()[name = tensor("op_755_dtype_0"), val = tensor("fp32")]; + tensor var_756 = const()[name = tensor("op_756"), val = tensor([1, 16, 1])]; + tensor var_755 = cast(dtype = var_755_dtype_0, x = var_754)[name = tensor("cast_133")]; + tensor fill_mask_11 = reshape(shape = var_756, x = var_755)[name = tensor("fill_mask_11")]; tensor x_expanded_11_reps_0 = const()[name = tensor("x_expanded_11_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_906)[name = tensor("x_expanded_11")]; - tensor var_920 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_920")]; - tensor var_921 = mul(x = input_121, y = var_920)[name = tensor("op_921")]; - tensor var_922 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_922")]; - tensor filled_window_11 = add(x = var_921, y = var_922)[name = tensor("filled_window_11")]; - tensor var_924 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_924")]; - tensor var_925_dtype_0 = const()[name = tensor("op_925_dtype_0"), val = tensor("fp32")]; - tensor var_926 = const()[name = tensor("op_926"), val = tensor([1, 1, 1])]; - tensor var_925 = cast(dtype = var_925_dtype_0, x = var_924)[name = tensor("cast_132")]; - tensor use_roll_11 = reshape(shape = var_926, x = var_925)[name = tensor("use_roll_11")]; - tensor var_928 = sub(x = var_17, y = use_roll_11)[name = tensor("op_928")]; - tensor var_929 = mul(x = filled_window_11, y = var_928)[name = tensor("op_929")]; - tensor var_930 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_930")]; - tensor input_131 = add(x = var_929, y = var_930)[name = tensor("input_131")]; - tensor var_932 = const()[name = tensor("op_932"), val = tensor(0x1p+0)]; - tensor var_933 = add(x = clip_9, y = var_932)[name = tensor("op_933")]; - tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_933)[name = tensor("clip_11")]; - tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; - tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_131)[name = tensor("x_23")]; - tensor input_133_perm_0 = const()[name = tensor("input_133_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_41_pad_type_0 = const()[name = tensor("inputs_41_pad_type_0"), val = tensor("valid")]; - tensor inputs_41_strides_0 = const()[name = tensor("inputs_41_strides_0"), val = tensor([1])]; - tensor inputs_41_pad_0 = const()[name = tensor("inputs_41_pad_0"), val = tensor([0, 0])]; - tensor inputs_41_dilations_0 = const()[name = tensor("inputs_41_dilations_0"), val = tensor([1])]; - tensor inputs_41_groups_0 = const()[name = tensor("inputs_41_groups_0"), val = tensor(1)]; - tensor input_133 = transpose(perm = input_133_perm_0, x = x_23)[name = tensor("transpose_40")]; - tensor inputs_41 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_41_dilations_0, groups = inputs_41_groups_0, pad = inputs_41_pad_0, pad_type = inputs_41_pad_type_0, strides = inputs_41_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_133)[name = tensor("inputs_41")]; - tensor var_954_split_sizes_0 = const()[name = tensor("op_954_split_sizes_0"), val = tensor([256, 256])]; - tensor var_954_axis_0 = const()[name = tensor("op_954_axis_0"), val = tensor(1)]; - tensor var_954_0, tensor var_954_1 = split(axis = var_954_axis_0, split_sizes = var_954_split_sizes_0, x = inputs_41)[name = tensor("op_954")]; - tensor var_956 = sigmoid(x = var_954_1)[name = tensor("op_956")]; - tensor inputs_43 = mul(x = var_954_0, y = var_956)[name = tensor("inputs_43")]; - tensor outputs_aug_11_pad_type_0 = const()[name = tensor("outputs_aug_11_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_11_pad_0 = const()[name = tensor("outputs_aug_11_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_11_groups_0 = const()[name = tensor("outputs_aug_11_groups_0"), val = tensor(256)]; - tensor outputs_aug_11_strides_0 = const()[name = tensor("outputs_aug_11_strides_0"), val = tensor([1])]; - tensor outputs_aug_11_dilations_0 = const()[name = tensor("outputs_aug_11_dilations_0"), val = tensor([1])]; - tensor outputs_aug_11 = conv(dilations = outputs_aug_11_dilations_0, groups = outputs_aug_11_groups_0, pad = outputs_aug_11_pad_0, pad_type = outputs_aug_11_pad_type_0, strides = outputs_aug_11_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_43)[name = tensor("outputs_aug_11")]; - tensor input_135_begin_0 = const()[name = tensor("input_135_begin_0"), val = tensor([0, 0, 0])]; - tensor input_135_end_0 = const()[name = tensor("input_135_end_0"), val = tensor([1, 256, 16])]; - tensor input_135_end_mask_0 = const()[name = tensor("input_135_end_mask_0"), val = tensor([true, true, false])]; - tensor input_135 = slice_by_index(begin = input_135_begin_0, end = input_135_end_0, end_mask = input_135_end_mask_0, x = outputs_aug_11)[name = tensor("input_135")]; - tensor inputs_45 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_135)[name = tensor("inputs_45")]; - tensor input_137 = silu(x = inputs_45)[name = tensor("input_137")]; - tensor input_139_pad_type_0 = const()[name = tensor("input_139_pad_type_0"), val = tensor("valid")]; - tensor input_139_strides_0 = const()[name = tensor("input_139_strides_0"), val = tensor([1])]; - tensor input_139_pad_0 = const()[name = tensor("input_139_pad_0"), val = tensor([0, 0])]; - tensor input_139_dilations_0 = const()[name = tensor("input_139_dilations_0"), val = tensor([1])]; - tensor input_139_groups_0 = const()[name = tensor("input_139_groups_0"), val = tensor(1)]; - tensor input_139 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_139_dilations_0, groups = input_139_groups_0, pad = input_139_pad_0, pad_type = input_139_pad_type_0, strides = input_139_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_137)[name = tensor("input_139")]; - tensor conv_out_11_perm_0 = const()[name = tensor("conv_out_11_perm_0"), val = tensor([0, 2, 1])]; - tensor var_987_begin_0 = const()[name = tensor("op_987_begin_0"), val = tensor([0, -1, 0])]; - tensor var_987_end_0 = const()[name = tensor("op_987_end_0"), val = tensor([1, 16, 256])]; - tensor var_987_end_mask_0 = const()[name = tensor("op_987_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_11 = transpose(perm = conv_out_11_perm_0, x = input_139)[name = tensor("transpose_39")]; - tensor var_987 = slice_by_index(begin = var_987_begin_0, end = var_987_end_0, end_mask = var_987_end_mask_0, x = conv_out_11)[name = tensor("op_987")]; - tensor var_990_interleave_0 = const()[name = tensor("op_990_interleave_0"), val = tensor(false)]; - tensor var_990 = concat(axis = var_33, interleave = var_990_interleave_0, values = (var_903, var_987))[name = tensor("op_990")]; - tensor input_141 = add(x = x_19, y = var_990)[name = tensor("input_141")]; - tensor input_143_axes_0 = const()[name = tensor("input_143_axes_0"), val = tensor([-1])]; - tensor input_143 = layer_norm(axes = input_143_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_141)[name = tensor("input_143")]; - tensor inputs_47 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_143)[name = tensor("linear_26")]; - tensor input_145 = silu(x = inputs_47)[name = tensor("input_145")]; - tensor input_149 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_145)[name = tensor("linear_27")]; - tensor var_1013 = const()[name = tensor("op_1013"), val = tensor(0x1p-1)]; - tensor var_1014 = mul(x = input_149, y = var_1013)[name = tensor("op_1014")]; - tensor input_151 = add(x = var_1014, y = input_141)[name = tensor("input_151")]; - tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; - tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_151)[name = tensor("input_153")]; - tensor input_155_axes_0 = const()[name = tensor("input_155_axes_0"), val = tensor([-1])]; - tensor input_155 = layer_norm(axes = input_155_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_153)[name = tensor("input_155")]; - tensor inputs_49 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_155)[name = tensor("linear_28")]; - tensor input_157 = silu(x = inputs_49)[name = tensor("input_157")]; - tensor input_161 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_157)[name = tensor("linear_29")]; - tensor var_1043 = const()[name = tensor("op_1043"), val = tensor(0x1p-1)]; - tensor var_1044 = mul(x = input_161, y = var_1043)[name = tensor("op_1044")]; - tensor input_163 = add(x = var_1044, y = input_153)[name = tensor("input_163")]; - tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; - tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_163)[name = tensor("x_25")]; + tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_746)[name = tensor("x_expanded_11")]; + tensor var_760 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_760")]; + tensor var_761 = mul(x = window_15, y = var_760)[name = tensor("op_761")]; + tensor var_762 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_762")]; + tensor filled_window_11 = add(x = var_761, y = var_762)[name = tensor("filled_window_11")]; + tensor var_764 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_764")]; + tensor var_765_dtype_0 = const()[name = tensor("op_765_dtype_0"), val = tensor("fp32")]; + tensor var_766 = const()[name = tensor("op_766"), val = tensor([1, 1, 1])]; + tensor var_765 = cast(dtype = var_765_dtype_0, x = var_764)[name = tensor("cast_132")]; + tensor use_roll_11 = reshape(shape = var_766, x = var_765)[name = tensor("use_roll_11")]; + tensor var_768 = sub(x = var_17, y = use_roll_11)[name = tensor("op_768")]; + tensor var_769 = mul(x = filled_window_11, y = var_768)[name = tensor("op_769")]; + tensor var_770 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_770")]; + tensor window_17 = add(x = var_769, y = var_770)[name = tensor("window_17")]; + tensor var_772 = const()[name = tensor("op_772"), val = tensor(0x1p+0)]; + tensor var_773 = add(x = clip_9, y = var_772)[name = tensor("op_773")]; + tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_773)[name = tensor("clip_11")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_30, interleave = input_101_interleave_0, values = (window_15, window_17))[name = tensor("input_101")]; + tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; + tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; + tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_23_pad_type_0 = const()[name = tensor("inputs_23_pad_type_0"), val = tensor("valid")]; + tensor inputs_23_strides_0 = const()[name = tensor("inputs_23_strides_0"), val = tensor([1])]; + tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; + tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; + tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; + tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; + tensor var_796_split_sizes_0 = const()[name = tensor("op_796_split_sizes_0"), val = tensor([256, 256])]; + tensor var_796_axis_0 = const()[name = tensor("op_796_axis_0"), val = tensor(1)]; + tensor var_796_0, tensor var_796_1 = split(axis = var_796_axis_0, split_sizes = var_796_split_sizes_0, x = inputs_23)[name = tensor("op_796")]; + tensor var_798 = sigmoid(x = var_796_1)[name = tensor("op_798")]; + tensor inputs_25 = mul(x = var_796_0, y = var_798)[name = tensor("inputs_25")]; + tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; + tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; + tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; + tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_25)[name = tensor("outputs_aug_5")]; + tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; + tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([2, 256, 16])]; + tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; + tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_5)[name = tensor("input_105")]; + tensor inputs_27 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_105)[name = tensor("inputs_27")]; + tensor input_107 = silu(x = inputs_27)[name = tensor("input_107")]; + tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; + tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; + tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; + tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; + tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; + tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; + tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; + tensor var_829_begin_0 = const()[name = tensor("op_829_begin_0"), val = tensor([0, -1, 0])]; + tensor var_829_end_0 = const()[name = tensor("op_829_end_0"), val = tensor([2, 16, 256])]; + tensor var_829_end_mask_0 = const()[name = tensor("op_829_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_829 = slice_by_index(begin = var_829_begin_0, end = var_829_end_0, end_mask = var_829_end_mask_0, x = conv_out_5)[name = tensor("op_829")]; + tensor var_831_perm_0 = const()[name = tensor("op_831_perm_0"), val = tensor([1, 0, 2])]; + tensor var_831 = transpose(perm = var_831_perm_0, x = var_829)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_831)[name = tensor("input_111")]; + tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; + tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; + tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; + tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; + tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; + tensor var_854 = const()[name = tensor("op_854"), val = tensor(0x1p-1)]; + tensor var_855 = mul(x = input_119, y = var_854)[name = tensor("op_855")]; + tensor input_121 = add(x = var_855, y = input_111)[name = tensor("input_121")]; + tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; + tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; + tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; + tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; + tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; + tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; + tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; + tensor var_884 = const()[name = tensor("op_884"), val = tensor(0x1p-1)]; + tensor var_885 = mul(x = input_131, y = var_884)[name = tensor("op_885")]; + tensor input_133 = add(x = var_885, y = input_123)[name = tensor("input_133")]; + tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; + tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; tensor prev_kv_7_end_0 = const()[name = tensor("prev_kv_7_end_0"), val = tensor([4, 1, 4, 64, 64])]; tensor prev_kv_7_end_mask_0 = const()[name = tensor("prev_kv_7_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -984,236 +874,199 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_1058 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_25)[name = tensor("linear_30")]; - tensor var_1059 = const()[name = tensor("op_1059"), val = tensor([1, 2, 4, 64])]; - tensor var_1060 = reshape(shape = var_1059, x = var_1058)[name = tensor("op_1060")]; + tensor var_899 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_900 = const()[name = tensor("op_900"), val = tensor([1, 2, 4, 64])]; + tensor var_901 = reshape(shape = var_900, x = var_899)[name = tensor("op_901")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1064 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_25)[name = tensor("linear_31")]; - tensor var_1065 = const()[name = tensor("op_1065"), val = tensor(0x1p-3)]; - tensor var_1066 = mul(x = var_1064, y = var_1065)[name = tensor("op_1066")]; - tensor var_1067 = const()[name = tensor("op_1067"), val = tensor([1, 2, 4, 64])]; - tensor var_1068 = reshape(shape = var_1067, x = var_1066)[name = tensor("op_1068")]; + tensor var_905 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_906 = const()[name = tensor("op_906"), val = tensor(0x1p-3)]; + tensor var_907 = mul(x = var_905, y = var_906)[name = tensor("op_907")]; + tensor var_908 = const()[name = tensor("op_908"), val = tensor([1, 2, 4, 64])]; + tensor var_909 = reshape(shape = var_908, x = var_907)[name = tensor("op_909")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1072 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_25)[name = tensor("linear_32")]; - tensor var_1073 = const()[name = tensor("op_1073"), val = tensor([1, 2, 4, 64])]; - tensor var_1074 = reshape(shape = var_1073, x = var_1072)[name = tensor("op_1074")]; + tensor var_913 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_914 = const()[name = tensor("op_914"), val = tensor([1, 2, 4, 64])]; + tensor var_915 = reshape(shape = var_914, x = var_913)[name = tensor("op_915")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_167 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_25)[name = tensor("linear_33")]; + tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; tensor s_t_7 = add(x = prev_scale_7, y = encoder__t_index)[name = tensor("s_t_7")]; tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_1068)[name = tensor("transpose_37")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_1060)[name = tensor("transpose_38")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_909)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_901)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_1084 = const()[name = tensor("op_1084"), val = tensor([2, 1])]; - tensor var_1085 = reshape(shape = var_1084, x = sqrt_s_t_7)[name = tensor("op_1085")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_1085)[name = tensor("M_7")]; - tensor var_1087 = mul(x = qk_7, y = M_7)[name = tensor("op_1087")]; + tensor var_925 = const()[name = tensor("op_925"), val = tensor([2, 1])]; + tensor var_926 = reshape(shape = var_925, x = sqrt_s_t_7)[name = tensor("op_926")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_926)[name = tensor("M_7")]; + tensor var_928 = mul(x = qk_7, y = M_7)[name = tensor("op_928")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_1074)[name = tensor("transpose_36")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1087, y = v_7)[name = tensor("inner_7")]; - tensor var_1089_transpose_x_0 = const()[name = tensor("op_1089_transpose_x_0"), val = tensor(false)]; - tensor var_1089_transpose_y_0 = const()[name = tensor("op_1089_transpose_y_0"), val = tensor(false)]; - tensor var_1089 = matmul(transpose_x = var_1089_transpose_x_0, transpose_y = var_1089_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1089")]; - tensor var_1090 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1090")]; - tensor var_1091 = const()[name = tensor("op_1091"), val = tensor([1, 1, 2, 1])]; - tensor var_1092 = reshape(shape = var_1091, x = var_1090)[name = tensor("op_1092")]; - tensor cross_7 = mul(x = var_1089, y = var_1092)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_915)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_928, y = v_7)[name = tensor("inner_7")]; + tensor var_930_transpose_x_0 = const()[name = tensor("op_930_transpose_x_0"), val = tensor(false)]; + tensor var_930_transpose_y_0 = const()[name = tensor("op_930_transpose_y_0"), val = tensor(false)]; + tensor var_930 = matmul(transpose_x = var_930_transpose_x_0, transpose_y = var_930_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_930")]; + tensor var_931 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_931")]; + tensor var_932 = const()[name = tensor("op_932"), val = tensor([1, 1, 2, 1])]; + tensor var_933 = reshape(shape = var_932, x = var_931)[name = tensor("op_933")]; + tensor cross_7 = mul(x = var_930, y = var_933)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_1095 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1095")]; - tensor var_1097_transpose_x_1 = const()[name = tensor("op_1097_transpose_x_1"), val = tensor(true)]; - tensor var_1097_transpose_y_1 = const()[name = tensor("op_1097_transpose_y_1"), val = tensor(false)]; - tensor var_1097 = matmul(transpose_x = var_1097_transpose_x_1, transpose_y = var_1097_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1097")]; - tensor new_kv_unnorm_7 = add(x = var_1095, y = var_1097)[name = tensor("new_kv_unnorm_7")]; - tensor var_1099 = const()[name = tensor("op_1099"), val = tensor(0x1p+1)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_1099)[name = tensor("new_scale_7")]; - tensor var_1101 = sqrt(x = new_scale_7)[name = tensor("op_1101")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1101)[name = tensor("nkv_1")]; - tensor var_1103_perm_0 = const()[name = tensor("op_1103_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_936 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_936")]; + tensor var_938_transpose_x_1 = const()[name = tensor("op_938_transpose_x_1"), val = tensor(true)]; + tensor var_938_transpose_y_1 = const()[name = tensor("op_938_transpose_y_1"), val = tensor(false)]; + tensor var_938 = matmul(transpose_x = var_938_transpose_x_1, transpose_y = var_938_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_938")]; + tensor new_kv_unnorm_7 = add(x = var_936, y = var_938)[name = tensor("new_kv_unnorm_7")]; + tensor var_940 = const()[name = tensor("op_940"), val = tensor(0x1p+1)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_940)[name = tensor("new_scale_7")]; + tensor var_942 = sqrt(x = new_scale_7)[name = tensor("op_942")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_942)[name = tensor("nkv_1")]; + tensor var_944_perm_0 = const()[name = tensor("op_944_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_1103 = transpose(perm = var_1103_perm_0, x = out_19)[name = tensor("transpose_35")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1103)[name = tensor("out_21")]; - tensor var_1107 = const()[name = tensor("op_1107"), val = tensor([1, 2, 256])]; - tensor out_23 = reshape(shape = var_1107, x = out_21)[name = tensor("out_23")]; - tensor var_1109 = silu(x = input_167)[name = tensor("op_1109")]; - tensor input_169 = mul(x = var_1109, y = out_23)[name = tensor("input_169")]; - tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_169)[name = tensor("linear_34")]; - tensor x_27 = add(x = input_163, y = ret_out_7)[name = tensor("x_27")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor var_944 = transpose(perm = var_944_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_944)[name = tensor("out_21")]; + tensor var_948 = const()[name = tensor("op_948"), val = tensor([1, 2, 256])]; + tensor out_23 = reshape(shape = var_948, x = out_21)[name = tensor("out_23")]; + tensor var_950 = silu(x = input_137)[name = tensor("op_950")]; + tensor input_139 = mul(x = var_950, y = out_23)[name = tensor("input_139")]; + tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; + tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; + tensor window_19_begin_0 = const()[name = tensor("window_19_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_19_end_0 = const()[name = tensor("window_19_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_19_end_mask_0 = const()[name = tensor("window_19_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_19_squeeze_mask_0 = const()[name = tensor("window_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_19 = slice_by_index(begin = window_19_begin_0, end = window_19_end_0, end_mask = window_19_end_mask_0, squeeze_mask = window_19_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_19")]; tensor filled_13_begin_0 = const()[name = tensor("filled_13_begin_0"), val = tensor([3, 0])]; tensor filled_13_end_0 = const()[name = tensor("filled_13_end_0"), val = tensor([4, 1])]; tensor filled_13_end_mask_0 = const()[name = tensor("filled_13_end_mask_0"), val = tensor([false, true])]; tensor filled_13_squeeze_mask_0 = const()[name = tensor("filled_13_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_13 = slice_by_index(begin = filled_13_begin_0, end = filled_13_end_0, end_mask = filled_13_end_mask_0, squeeze_mask = filled_13_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_13")]; - tensor var_1118_begin_0 = const()[name = tensor("op_1118_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1118_end_0 = const()[name = tensor("op_1118_end_0"), val = tensor([1, 1, 256])]; - tensor var_1118_end_mask_0 = const()[name = tensor("op_1118_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1118 = slice_by_index(begin = var_1118_begin_0, end = var_1118_end_0, end_mask = var_1118_end_mask_0, x = x_27)[name = tensor("op_1118")]; - tensor var_1121_begin_0 = const()[name = tensor("op_1121_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1121_end_0 = const()[name = tensor("op_1121_end_0"), val = tensor([1, 16, 256])]; - tensor var_1121_end_mask_0 = const()[name = tensor("op_1121_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1121 = slice_by_index(begin = var_1121_begin_0, end = var_1121_end_0, end_mask = var_1121_end_mask_0, x = window)[name = tensor("op_1121")]; + tensor var_959_begin_0 = const()[name = tensor("op_959_begin_0"), val = tensor([0, 0, 0])]; + tensor var_959_end_0 = const()[name = tensor("op_959_end_0"), val = tensor([1, 1, 256])]; + tensor var_959_end_mask_0 = const()[name = tensor("op_959_end_mask_0"), val = tensor([true, false, true])]; + tensor var_959 = slice_by_index(begin = var_959_begin_0, end = var_959_end_0, end_mask = var_959_end_mask_0, x = x_21)[name = tensor("op_959")]; + tensor var_962_begin_0 = const()[name = tensor("op_962_begin_0"), val = tensor([0, 1, 0])]; + tensor var_962_end_0 = const()[name = tensor("op_962_end_0"), val = tensor([1, 16, 256])]; + tensor var_962_end_mask_0 = const()[name = tensor("op_962_end_mask_0"), val = tensor([true, true, true])]; + tensor var_962 = slice_by_index(begin = var_962_begin_0, end = var_962_end_0, end_mask = var_962_end_mask_0, x = window_19)[name = tensor("op_962")]; tensor rolled_13_interleave_0 = const()[name = tensor("rolled_13_interleave_0"), val = tensor(false)]; - tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_1121, var_1118))[name = tensor("rolled_13")]; + tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_962, var_959))[name = tensor("rolled_13")]; tensor clip_12 = clip(alpha = var_21, beta = var_20, x = filled_13)[name = tensor("clip_12")]; - tensor var_1126 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_1126")]; - tensor var_1127_dtype_0 = const()[name = tensor("op_1127_dtype_0"), val = tensor("fp32")]; - tensor var_1128 = const()[name = tensor("op_1128"), val = tensor([1, 16, 1])]; - tensor var_1127 = cast(dtype = var_1127_dtype_0, x = var_1126)[name = tensor("cast_131")]; - tensor fill_mask_13 = reshape(shape = var_1128, x = var_1127)[name = tensor("fill_mask_13")]; + tensor var_967 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_967")]; + tensor var_968_dtype_0 = const()[name = tensor("op_968_dtype_0"), val = tensor("fp32")]; + tensor var_969 = const()[name = tensor("op_969"), val = tensor([1, 16, 1])]; + tensor var_968 = cast(dtype = var_968_dtype_0, x = var_967)[name = tensor("cast_131")]; + tensor fill_mask_13 = reshape(shape = var_969, x = var_968)[name = tensor("fill_mask_13")]; tensor x_expanded_13_reps_0 = const()[name = tensor("x_expanded_13_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_1118)[name = tensor("x_expanded_13")]; - tensor var_1132 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_1132")]; - tensor var_1133 = mul(x = window, y = var_1132)[name = tensor("op_1133")]; - tensor var_1134 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_1134")]; - tensor filled_window_13 = add(x = var_1133, y = var_1134)[name = tensor("filled_window_13")]; - tensor var_1136 = greater_equal(x = filled_13, y = var_16)[name = tensor("op_1136")]; - tensor var_1137_dtype_0 = const()[name = tensor("op_1137_dtype_0"), val = tensor("fp32")]; - tensor var_1138 = const()[name = tensor("op_1138"), val = tensor([1, 1, 1])]; - tensor var_1137 = cast(dtype = var_1137_dtype_0, x = var_1136)[name = tensor("cast_130")]; - tensor use_roll_13 = reshape(shape = var_1138, x = var_1137)[name = tensor("use_roll_13")]; - tensor var_1140 = sub(x = var_17, y = use_roll_13)[name = tensor("op_1140")]; - tensor var_1141 = mul(x = filled_window_13, y = var_1140)[name = tensor("op_1141")]; - tensor var_1142 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_1142")]; - tensor input_171 = add(x = var_1141, y = var_1142)[name = tensor("input_171")]; - tensor var_1144 = const()[name = tensor("op_1144"), val = tensor(0x1p+0)]; - tensor var_1145 = add(x = filled_13, y = var_1144)[name = tensor("op_1145")]; - tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_1145)[name = tensor("clip_13")]; - tensor x_29_axes_0 = const()[name = tensor("x_29_axes_0"), val = tensor([-1])]; - tensor x_29 = layer_norm(axes = x_29_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_171)[name = tensor("x_29")]; - tensor input_173_perm_0 = const()[name = tensor("input_173_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_51_pad_type_0 = const()[name = tensor("inputs_51_pad_type_0"), val = tensor("valid")]; - tensor inputs_51_strides_0 = const()[name = tensor("inputs_51_strides_0"), val = tensor([1])]; - tensor inputs_51_pad_0 = const()[name = tensor("inputs_51_pad_0"), val = tensor([0, 0])]; - tensor inputs_51_dilations_0 = const()[name = tensor("inputs_51_dilations_0"), val = tensor([1])]; - tensor inputs_51_groups_0 = const()[name = tensor("inputs_51_groups_0"), val = tensor(1)]; - tensor input_173 = transpose(perm = input_173_perm_0, x = x_29)[name = tensor("transpose_34")]; - tensor inputs_51 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_51_dilations_0, groups = inputs_51_groups_0, pad = inputs_51_pad_0, pad_type = inputs_51_pad_type_0, strides = inputs_51_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_173)[name = tensor("inputs_51")]; - tensor var_1166_split_sizes_0 = const()[name = tensor("op_1166_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1166_axis_0 = const()[name = tensor("op_1166_axis_0"), val = tensor(1)]; - tensor var_1166_0, tensor var_1166_1 = split(axis = var_1166_axis_0, split_sizes = var_1166_split_sizes_0, x = inputs_51)[name = tensor("op_1166")]; - tensor var_1168 = sigmoid(x = var_1166_1)[name = tensor("op_1168")]; - tensor inputs_53 = mul(x = var_1166_0, y = var_1168)[name = tensor("inputs_53")]; - tensor outputs_aug_13_pad_type_0 = const()[name = tensor("outputs_aug_13_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_13_pad_0 = const()[name = tensor("outputs_aug_13_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_13_groups_0 = const()[name = tensor("outputs_aug_13_groups_0"), val = tensor(256)]; - tensor outputs_aug_13_strides_0 = const()[name = tensor("outputs_aug_13_strides_0"), val = tensor([1])]; - tensor outputs_aug_13_dilations_0 = const()[name = tensor("outputs_aug_13_dilations_0"), val = tensor([1])]; - tensor outputs_aug_13 = conv(dilations = outputs_aug_13_dilations_0, groups = outputs_aug_13_groups_0, pad = outputs_aug_13_pad_0, pad_type = outputs_aug_13_pad_type_0, strides = outputs_aug_13_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_53)[name = tensor("outputs_aug_13")]; - tensor input_175_begin_0 = const()[name = tensor("input_175_begin_0"), val = tensor([0, 0, 0])]; - tensor input_175_end_0 = const()[name = tensor("input_175_end_0"), val = tensor([1, 256, 16])]; - tensor input_175_end_mask_0 = const()[name = tensor("input_175_end_mask_0"), val = tensor([true, true, false])]; - tensor input_175 = slice_by_index(begin = input_175_begin_0, end = input_175_end_0, end_mask = input_175_end_mask_0, x = outputs_aug_13)[name = tensor("input_175")]; - tensor inputs_55 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_175)[name = tensor("inputs_55")]; - tensor input_177 = silu(x = inputs_55)[name = tensor("input_177")]; - tensor input_179_pad_type_0 = const()[name = tensor("input_179_pad_type_0"), val = tensor("valid")]; - tensor input_179_strides_0 = const()[name = tensor("input_179_strides_0"), val = tensor([1])]; - tensor input_179_pad_0 = const()[name = tensor("input_179_pad_0"), val = tensor([0, 0])]; - tensor input_179_dilations_0 = const()[name = tensor("input_179_dilations_0"), val = tensor([1])]; - tensor input_179_groups_0 = const()[name = tensor("input_179_groups_0"), val = tensor(1)]; - tensor input_179 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_179_dilations_0, groups = input_179_groups_0, pad = input_179_pad_0, pad_type = input_179_pad_type_0, strides = input_179_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_177)[name = tensor("input_179")]; - tensor conv_out_13_perm_0 = const()[name = tensor("conv_out_13_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1199_begin_0 = const()[name = tensor("op_1199_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1199_end_0 = const()[name = tensor("op_1199_end_0"), val = tensor([1, 16, 256])]; - tensor var_1199_end_mask_0 = const()[name = tensor("op_1199_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_13 = transpose(perm = conv_out_13_perm_0, x = input_179)[name = tensor("transpose_33")]; - tensor var_1199 = slice_by_index(begin = var_1199_begin_0, end = var_1199_end_0, end_mask = var_1199_end_mask_0, x = conv_out_13)[name = tensor("op_1199")]; - tensor var_1202_begin_0 = const()[name = tensor("op_1202_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1202_end_0 = const()[name = tensor("op_1202_end_0"), val = tensor([1, 1, 256])]; - tensor var_1202_end_mask_0 = const()[name = tensor("op_1202_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1202 = slice_by_index(begin = var_1202_begin_0, end = var_1202_end_0, end_mask = var_1202_end_mask_0, x = x_27)[name = tensor("op_1202")]; - tensor var_1205_begin_0 = const()[name = tensor("op_1205_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1205_end_0 = const()[name = tensor("op_1205_end_0"), val = tensor([1, 16, 256])]; - tensor var_1205_end_mask_0 = const()[name = tensor("op_1205_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1205 = slice_by_index(begin = var_1205_begin_0, end = var_1205_end_0, end_mask = var_1205_end_mask_0, x = input_171)[name = tensor("op_1205")]; + tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_959)[name = tensor("x_expanded_13")]; + tensor var_973 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_973")]; + tensor var_974 = mul(x = window_19, y = var_973)[name = tensor("op_974")]; + tensor var_975 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_975")]; + tensor filled_window_13 = add(x = var_974, y = var_975)[name = tensor("filled_window_13")]; + tensor var_977 = greater_equal(x = filled_13, y = var_16)[name = tensor("op_977")]; + tensor var_978_dtype_0 = const()[name = tensor("op_978_dtype_0"), val = tensor("fp32")]; + tensor var_979 = const()[name = tensor("op_979"), val = tensor([1, 1, 1])]; + tensor var_978 = cast(dtype = var_978_dtype_0, x = var_977)[name = tensor("cast_130")]; + tensor use_roll_13 = reshape(shape = var_979, x = var_978)[name = tensor("use_roll_13")]; + tensor var_981 = sub(x = var_17, y = use_roll_13)[name = tensor("op_981")]; + tensor var_982 = mul(x = filled_window_13, y = var_981)[name = tensor("op_982")]; + tensor var_983 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_983")]; + tensor window_21 = add(x = var_982, y = var_983)[name = tensor("window_21")]; + tensor var_985 = const()[name = tensor("op_985"), val = tensor(0x1p+0)]; + tensor var_986 = add(x = filled_13, y = var_985)[name = tensor("op_986")]; + tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_986)[name = tensor("clip_13")]; + tensor var_989_begin_0 = const()[name = tensor("op_989_begin_0"), val = tensor([0, 1, 0])]; + tensor var_989_end_0 = const()[name = tensor("op_989_end_0"), val = tensor([1, 1, 256])]; + tensor var_989_end_mask_0 = const()[name = tensor("op_989_end_mask_0"), val = tensor([true, true, true])]; + tensor var_989 = slice_by_index(begin = var_989_begin_0, end = var_989_end_0, end_mask = var_989_end_mask_0, x = x_21)[name = tensor("op_989")]; + tensor var_992_begin_0 = const()[name = tensor("op_992_begin_0"), val = tensor([0, 1, 0])]; + tensor var_992_end_0 = const()[name = tensor("op_992_end_0"), val = tensor([1, 16, 256])]; + tensor var_992_end_mask_0 = const()[name = tensor("op_992_end_mask_0"), val = tensor([true, true, true])]; + tensor var_992 = slice_by_index(begin = var_992_begin_0, end = var_992_end_0, end_mask = var_992_end_mask_0, x = window_21)[name = tensor("op_992")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1205, var_1202))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_992, var_989))[name = tensor("rolled")]; tensor clip_14 = clip(alpha = var_21, beta = var_20, x = clip_13)[name = tensor("clip_14")]; - tensor var_1210 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_1210")]; - tensor var_1211_dtype_0 = const()[name = tensor("op_1211_dtype_0"), val = tensor("fp32")]; - tensor var_1212 = const()[name = tensor("op_1212"), val = tensor([1, 16, 1])]; - tensor var_1211 = cast(dtype = var_1211_dtype_0, x = var_1210)[name = tensor("cast_129")]; - tensor fill_mask = reshape(shape = var_1212, x = var_1211)[name = tensor("fill_mask")]; + tensor var_997 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_997")]; + tensor var_998_dtype_0 = const()[name = tensor("op_998_dtype_0"), val = tensor("fp32")]; + tensor var_999 = const()[name = tensor("op_999"), val = tensor([1, 16, 1])]; + tensor var_998 = cast(dtype = var_998_dtype_0, x = var_997)[name = tensor("cast_129")]; + tensor fill_mask = reshape(shape = var_999, x = var_998)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1202)[name = tensor("x_expanded")]; - tensor var_1216 = sub(x = var_17, y = fill_mask)[name = tensor("op_1216")]; - tensor var_1217 = mul(x = input_171, y = var_1216)[name = tensor("op_1217")]; - tensor var_1218 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1218")]; - tensor filled_window = add(x = var_1217, y = var_1218)[name = tensor("filled_window")]; - tensor var_1220 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_1220")]; - tensor var_1221_dtype_0 = const()[name = tensor("op_1221_dtype_0"), val = tensor("fp32")]; - tensor var_1222 = const()[name = tensor("op_1222"), val = tensor([1, 1, 1])]; - tensor var_1221 = cast(dtype = var_1221_dtype_0, x = var_1220)[name = tensor("cast_128")]; - tensor use_roll = reshape(shape = var_1222, x = var_1221)[name = tensor("use_roll")]; - tensor var_1224 = sub(x = var_17, y = use_roll)[name = tensor("op_1224")]; - tensor var_1225 = mul(x = filled_window, y = var_1224)[name = tensor("op_1225")]; - tensor var_1226 = mul(x = rolled, y = use_roll)[name = tensor("op_1226")]; - tensor input_181 = add(x = var_1225, y = var_1226)[name = tensor("input_181")]; - tensor var_1228 = const()[name = tensor("op_1228"), val = tensor(0x1p+0)]; - tensor var_1229 = add(x = clip_13, y = var_1228)[name = tensor("op_1229")]; - tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_1229)[name = tensor("clip_15")]; - tensor x_31_axes_0 = const()[name = tensor("x_31_axes_0"), val = tensor([-1])]; - tensor x_31 = layer_norm(axes = x_31_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_181)[name = tensor("x_31")]; - tensor input_183_perm_0 = const()[name = tensor("input_183_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_57_pad_type_0 = const()[name = tensor("inputs_57_pad_type_0"), val = tensor("valid")]; - tensor inputs_57_strides_0 = const()[name = tensor("inputs_57_strides_0"), val = tensor([1])]; - tensor inputs_57_pad_0 = const()[name = tensor("inputs_57_pad_0"), val = tensor([0, 0])]; - tensor inputs_57_dilations_0 = const()[name = tensor("inputs_57_dilations_0"), val = tensor([1])]; - tensor inputs_57_groups_0 = const()[name = tensor("inputs_57_groups_0"), val = tensor(1)]; - tensor input_183 = transpose(perm = input_183_perm_0, x = x_31)[name = tensor("transpose_32")]; - tensor inputs_57 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_57_dilations_0, groups = inputs_57_groups_0, pad = inputs_57_pad_0, pad_type = inputs_57_pad_type_0, strides = inputs_57_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_183)[name = tensor("inputs_57")]; - tensor var_1250_split_sizes_0 = const()[name = tensor("op_1250_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1250_axis_0 = const()[name = tensor("op_1250_axis_0"), val = tensor(1)]; - tensor var_1250_0, tensor var_1250_1 = split(axis = var_1250_axis_0, split_sizes = var_1250_split_sizes_0, x = inputs_57)[name = tensor("op_1250")]; - tensor var_1252 = sigmoid(x = var_1250_1)[name = tensor("op_1252")]; - tensor inputs_59 = mul(x = var_1250_0, y = var_1252)[name = tensor("inputs_59")]; + tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_989)[name = tensor("x_expanded")]; + tensor var_1003 = sub(x = var_17, y = fill_mask)[name = tensor("op_1003")]; + tensor var_1004 = mul(x = window_21, y = var_1003)[name = tensor("op_1004")]; + tensor var_1005 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1005")]; + tensor filled_window = add(x = var_1004, y = var_1005)[name = tensor("filled_window")]; + tensor var_1007 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_1007")]; + tensor var_1008_dtype_0 = const()[name = tensor("op_1008_dtype_0"), val = tensor("fp32")]; + tensor var_1009 = const()[name = tensor("op_1009"), val = tensor([1, 1, 1])]; + tensor var_1008 = cast(dtype = var_1008_dtype_0, x = var_1007)[name = tensor("cast_128")]; + tensor use_roll = reshape(shape = var_1009, x = var_1008)[name = tensor("use_roll")]; + tensor var_1011 = sub(x = var_17, y = use_roll)[name = tensor("op_1011")]; + tensor var_1012 = mul(x = filled_window, y = var_1011)[name = tensor("op_1012")]; + tensor var_1013 = mul(x = rolled, y = use_roll)[name = tensor("op_1013")]; + tensor window = add(x = var_1012, y = var_1013)[name = tensor("window")]; + tensor var_1015 = const()[name = tensor("op_1015"), val = tensor(0x1p+0)]; + tensor var_1016 = add(x = clip_13, y = var_1015)[name = tensor("op_1016")]; + tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_1016)[name = tensor("clip_15")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_30, interleave = input_141_interleave_0, values = (window_21, window))[name = tensor("input_141")]; + tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; + tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; + tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_33_pad_type_0 = const()[name = tensor("inputs_33_pad_type_0"), val = tensor("valid")]; + tensor inputs_33_strides_0 = const()[name = tensor("inputs_33_strides_0"), val = tensor([1])]; + tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; + tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; + tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; + tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; + tensor var_1039_split_sizes_0 = const()[name = tensor("op_1039_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1039_axis_0 = const()[name = tensor("op_1039_axis_0"), val = tensor(1)]; + tensor var_1039_0, tensor var_1039_1 = split(axis = var_1039_axis_0, split_sizes = var_1039_split_sizes_0, x = inputs_33)[name = tensor("op_1039")]; + tensor var_1041 = sigmoid(x = var_1039_1)[name = tensor("op_1041")]; + tensor inputs_35 = mul(x = var_1039_0, y = var_1041)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; tensor outputs_aug_strides_0 = const()[name = tensor("outputs_aug_strides_0"), val = tensor([1])]; tensor outputs_aug_dilations_0 = const()[name = tensor("outputs_aug_dilations_0"), val = tensor([1])]; - tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_59)[name = tensor("outputs_aug")]; - tensor input_185_begin_0 = const()[name = tensor("input_185_begin_0"), val = tensor([0, 0, 0])]; - tensor input_185_end_0 = const()[name = tensor("input_185_end_0"), val = tensor([1, 256, 16])]; - tensor input_185_end_mask_0 = const()[name = tensor("input_185_end_mask_0"), val = tensor([true, true, false])]; - tensor input_185 = slice_by_index(begin = input_185_begin_0, end = input_185_end_0, end_mask = input_185_end_mask_0, x = outputs_aug)[name = tensor("input_185")]; - tensor inputs_61 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_185)[name = tensor("inputs_61")]; - tensor input_187 = silu(x = inputs_61)[name = tensor("input_187")]; - tensor input_189_pad_type_0 = const()[name = tensor("input_189_pad_type_0"), val = tensor("valid")]; - tensor input_189_strides_0 = const()[name = tensor("input_189_strides_0"), val = tensor([1])]; - tensor input_189_pad_0 = const()[name = tensor("input_189_pad_0"), val = tensor([0, 0])]; - tensor input_189_dilations_0 = const()[name = tensor("input_189_dilations_0"), val = tensor([1])]; - tensor input_189_groups_0 = const()[name = tensor("input_189_groups_0"), val = tensor(1)]; - tensor input_189 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_189_dilations_0, groups = input_189_groups_0, pad = input_189_pad_0, pad_type = input_189_pad_type_0, strides = input_189_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_187)[name = tensor("input_189")]; - tensor conv_out_15_perm_0 = const()[name = tensor("conv_out_15_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1283_begin_0 = const()[name = tensor("op_1283_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1283_end_0 = const()[name = tensor("op_1283_end_0"), val = tensor([1, 16, 256])]; - tensor var_1283_end_mask_0 = const()[name = tensor("op_1283_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_15 = transpose(perm = conv_out_15_perm_0, x = input_189)[name = tensor("transpose_31")]; - tensor var_1283 = slice_by_index(begin = var_1283_begin_0, end = var_1283_end_0, end_mask = var_1283_end_mask_0, x = conv_out_15)[name = tensor("op_1283")]; - tensor var_1286_interleave_0 = const()[name = tensor("op_1286_interleave_0"), val = tensor(false)]; - tensor var_1286 = concat(axis = var_33, interleave = var_1286_interleave_0, values = (var_1199, var_1283))[name = tensor("op_1286")]; - tensor input_191 = add(x = x_27, y = var_1286)[name = tensor("input_191")]; - tensor input_193_axes_0 = const()[name = tensor("input_193_axes_0"), val = tensor([-1])]; - tensor input_193 = layer_norm(axes = input_193_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_191)[name = tensor("input_193")]; - tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_193)[name = tensor("linear_35")]; - tensor input_195 = silu(x = inputs)[name = tensor("input_195")]; - tensor input_199 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_195)[name = tensor("linear_36")]; - tensor var_1309 = const()[name = tensor("op_1309"), val = tensor(0x1p-1)]; - tensor var_1310 = mul(x = input_199, y = var_1309)[name = tensor("op_1310")]; - tensor input_201 = add(x = var_1310, y = input_191)[name = tensor("input_201")]; - tensor x_33_axes_0 = const()[name = tensor("x_33_axes_0"), val = tensor([-1])]; - tensor x_33 = layer_norm(axes = x_33_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_201)[name = tensor("x_33")]; + tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_35)[name = tensor("outputs_aug")]; + tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; + tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([2, 256, 16])]; + tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; + tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug)[name = tensor("input_145")]; + tensor inputs_37 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_145)[name = tensor("inputs_37")]; + tensor input_147 = silu(x = inputs_37)[name = tensor("input_147")]; + tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; + tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; + tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; + tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; + tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; + tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; + tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1072_begin_0 = const()[name = tensor("op_1072_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1072_end_0 = const()[name = tensor("op_1072_end_0"), val = tensor([2, 16, 256])]; + tensor var_1072_end_mask_0 = const()[name = tensor("op_1072_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_1072 = slice_by_index(begin = var_1072_begin_0, end = var_1072_end_0, end_mask = var_1072_end_mask_0, x = conv_out_7)[name = tensor("op_1072")]; + tensor var_1074_perm_0 = const()[name = tensor("op_1074_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1074 = transpose(perm = var_1074_perm_0, x = var_1072)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_1074)[name = tensor("input_151")]; + tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; + tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; + tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; + tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; + tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; + tensor var_1097 = const()[name = tensor("op_1097"), val = tensor(0x1p-1)]; + tensor var_1098 = mul(x = input_159, y = var_1097)[name = tensor("op_1098")]; + tensor input_161 = add(x = var_1098, y = input_151)[name = tensor("input_161")]; + tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; + tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; tensor cat_interleave_0 = const()[name = tensor("cat_interleave_0"), val = tensor(false)]; - tensor x_ct = transpose(perm = x_ct_perm_0, x = x_33)[name = tensor("transpose_30")]; + tensor x_ct = transpose(perm = x_ct_perm_0, x = x_25)[name = tensor("transpose_30")]; tensor cat = concat(axis = var_27, interleave = cat_interleave_0, values = (cnn_window, x_ct))[name = tensor("cat")]; tensor conv_out_pad_type_0 = const()[name = tensor("conv_out_pad_type_0"), val = tensor("valid")]; tensor conv_out_strides_0 = const()[name = tensor("conv_out_strides_0"), val = tensor([1])]; @@ -1221,44 +1074,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_1328_begin_0 = const()[name = tensor("op_1328_begin_0"), val = tensor([0, 0, 2])]; - tensor var_1328_end_0 = const()[name = tensor("op_1328_end_0"), val = tensor([1, 256, 20])]; - tensor var_1328_end_mask_0 = const()[name = tensor("op_1328_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_1328_begin_0, end = var_1328_end_0, end_mask = var_1328_end_mask_0, x = cat)[name = tensor("op_1328")]; - tensor input_203_perm_0 = const()[name = tensor("input_203_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1330 = const()[name = tensor("op_1330"), val = tensor([-1])]; - tensor input_203 = transpose(perm = input_203_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_1331 = reduce_l2_norm(axes = var_1330, keep_dims = var_36, x = input_203)[name = tensor("op_1331")]; - tensor const_16 = const()[name = tensor("const_16"), val = tensor(0x1.fffffep+127)]; - tensor clip_16 = clip(alpha = var_12, beta = const_16, x = var_1331)[name = tensor("clip_16")]; - tensor emb = real_div(x = input_203, y = clip_16)[name = tensor("emb")]; - tensor var_1335_axis_0 = const()[name = tensor("op_1335_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_1335_axis_0, values = (var_214, var_510, var_806, nkv_1))[name = tensor("op_1335")]; - tensor var_1337_axis_0 = const()[name = tensor("op_1337_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_1337_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1337")]; - tensor var_1339_axis_0 = const()[name = tensor("op_1339_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_1339_axis_0, values = (input_31, input_81, input_131, input_181))[name = tensor("op_1339")]; - tensor var_1341_axis_0 = const()[name = tensor("op_1341_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_1341_axis_0, values = (clip_3, clip_7, clip_11, clip_15))[name = tensor("op_1341")]; - tensor var_1351 = const()[name = tensor("op_1351"), val = tensor(0x1.5798eep-27)]; - tensor var_1357 = const()[name = tensor("op_1357"), val = tensor(0x1.4f8b58p-17)]; - tensor var_1359 = const()[name = tensor("op_1359"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_1360 = const()[name = tensor("op_1360"), val = tensor(true)]; - tensor var_1362 = const()[name = tensor("op_1362"), val = tensor(0x1p+0)]; - tensor var_1366 = const()[name = tensor("op_1366"), val = tensor(-1)]; - tensor var_1371 = const()[name = tensor("op_1371"), val = tensor(0)]; + tensor var_1116_begin_0 = const()[name = tensor("op_1116_begin_0"), val = tensor([0, 0, 2])]; + tensor var_1116_end_0 = const()[name = tensor("op_1116_end_0"), val = tensor([1, 256, 20])]; + tensor var_1116_end_mask_0 = const()[name = tensor("op_1116_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_1116_begin_0, end = var_1116_end_0, end_mask = var_1116_end_mask_0, x = cat)[name = tensor("op_1116")]; + tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1118 = const()[name = tensor("op_1118"), val = tensor([-1])]; + tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; + tensor var_1119 = reduce_l2_norm(axes = var_1118, keep_dims = var_36, x = input_163)[name = tensor("op_1119")]; + tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; + tensor clip_16 = clip(alpha = var_12, beta = const_12, x = var_1119)[name = tensor("clip_16")]; + tensor emb = real_div(x = input_163, y = clip_16)[name = tensor("emb")]; + tensor var_1123_axis_0 = const()[name = tensor("op_1123_axis_0"), val = tensor(0)]; + tensor enc_kv_new = stack(axis = var_1123_axis_0, values = (var_214, var_457, var_700, nkv_1))[name = tensor("op_1123")]; + tensor var_1125_axis_0 = const()[name = tensor("op_1125_axis_0"), val = tensor(0)]; + tensor enc_scale_new = stack(axis = var_1125_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1125")]; + tensor var_1127_axis_0 = const()[name = tensor("op_1127_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1127_axis_0, values = (window_5, window_11, window_17, window))[name = tensor("op_1127")]; + tensor var_1129_axis_0 = const()[name = tensor("op_1129_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1129_axis_0, values = (clip_3, clip_7, clip_11, clip_15))[name = tensor("op_1129")]; + tensor var_1139 = const()[name = tensor("op_1139"), val = tensor(0x1.5798eep-27)]; + tensor var_1144 = const()[name = tensor("op_1144"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1146 = const()[name = tensor("op_1146"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1147 = const()[name = tensor("op_1147"), val = tensor(true)]; + tensor var_1149 = const()[name = tensor("op_1149"), val = tensor(0x1p+0)]; + tensor var_1153 = const()[name = tensor("op_1153"), val = tensor(-1)]; + tensor var_1159 = const()[name = tensor("op_1159"), val = tensor(0)]; tensor pos = const()[name = tensor("pos"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395456)))]; - tensor var_1433_axes_0 = const()[name = tensor("op_1433_axes_0"), val = tensor([2])]; - tensor var_1433 = expand_dims(axes = var_1433_axes_0, x = emb)[name = tensor("op_1433")]; - tensor var_1434 = const()[name = tensor("op_1434"), val = tensor([1, 1, 12, 1])]; - tensor emb_exp = tile(reps = var_1434, x = var_1433)[name = tensor("emb_exp")]; - tensor input_205_interleave_0 = const()[name = tensor("input_205_interleave_0"), val = tensor(false)]; - tensor input_205 = concat(axis = var_1366, interleave = input_205_interleave_0, values = (emb_exp, pos))[name = tensor("input_205")]; - tensor x_35 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_205)[name = tensor("linear_37")]; - tensor var_1441_perm_0 = const()[name = tensor("op_1441_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1445 = const()[name = tensor("op_1445"), val = tensor([12, 2, 256])]; - tensor var_1441 = transpose(perm = var_1441_perm_0, x = x_35)[name = tensor("transpose_28")]; - tensor x_37 = reshape(shape = var_1445, x = var_1441)[name = tensor("x_37")]; + tensor var_1221_axes_0 = const()[name = tensor("op_1221_axes_0"), val = tensor([2])]; + tensor var_1221 = expand_dims(axes = var_1221_axes_0, x = emb)[name = tensor("op_1221")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 12, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1221)[name = tensor("emb_exp")]; + tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; + tensor input_165 = concat(axis = var_1153, interleave = input_165_interleave_0, values = (emb_exp, pos))[name = tensor("input_165")]; + tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; + tensor var_1229_perm_0 = const()[name = tensor("op_1229_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1233 = const()[name = tensor("op_1233"), val = tensor([12, 2, 256])]; + tensor var_1229 = transpose(perm = var_1229_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1233, x = var_1229)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 12, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1269,132 +1122,132 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_1453 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_37)[name = tensor("linear_38")]; - tensor var_1454 = const()[name = tensor("op_1454"), val = tensor([12, 2, 4, 64])]; - tensor var_1455 = reshape(shape = var_1454, x = var_1453)[name = tensor("op_1455")]; + tensor var_1241 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1242 = const()[name = tensor("op_1242"), val = tensor([12, 2, 4, 64])]; + tensor var_1243 = reshape(shape = var_1242, x = var_1241)[name = tensor("op_1243")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1459 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_37)[name = tensor("linear_39")]; - tensor var_1460 = const()[name = tensor("op_1460"), val = tensor(0x1p-3)]; - tensor var_1461 = mul(x = var_1459, y = var_1460)[name = tensor("op_1461")]; - tensor var_1462 = const()[name = tensor("op_1462"), val = tensor([12, 2, 4, 64])]; - tensor var_1463 = reshape(shape = var_1462, x = var_1461)[name = tensor("op_1463")]; + tensor var_1247 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1248 = const()[name = tensor("op_1248"), val = tensor(0x1p-3)]; + tensor var_1249 = mul(x = var_1247, y = var_1248)[name = tensor("op_1249")]; + tensor var_1250 = const()[name = tensor("op_1250"), val = tensor([12, 2, 4, 64])]; + tensor var_1251 = reshape(shape = var_1250, x = var_1249)[name = tensor("op_1251")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1467 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_37)[name = tensor("linear_40")]; - tensor var_1468 = const()[name = tensor("op_1468"), val = tensor([12, 2, 4, 64])]; - tensor var_1469 = reshape(shape = var_1468, x = var_1467)[name = tensor("op_1469")]; + tensor var_1255 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1256 = const()[name = tensor("op_1256"), val = tensor([12, 2, 4, 64])]; + tensor var_1257 = reshape(shape = var_1256, x = var_1255)[name = tensor("op_1257")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_209 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_37)[name = tensor("linear_41")]; + tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_1371, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1159, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; - tensor const_24 = const()[name = tensor("const_24"), val = tensor(0x1.fffffep+127)]; - tensor clip_17 = clip(alpha = var_1362, beta = const_24, x = s_t_9)[name = tensor("clip_17")]; + tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; + tensor clip_17 = clip(alpha = var_1149, beta = const_20, x = s_t_9)[name = tensor("clip_17")]; tensor sqrt_s_t_9 = sqrt(x = clip_17)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_1463)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_1455)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1251)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1243)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_1481 = const()[name = tensor("op_1481"), val = tensor([1, 2])]; - tensor var_1482 = reshape(shape = var_1481, x = valid_mask)[name = tensor("op_1482")]; - tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1482)[name = tensor("causal_with_valid_1")]; - tensor var_1484 = const()[name = tensor("op_1484"), val = tensor([2, 1])]; - tensor var_1485 = reshape(shape = var_1484, x = sqrt_s_t_9)[name = tensor("op_1485")]; - tensor M_9 = real_div(x = causal_with_valid_1, y = var_1485)[name = tensor("M_9")]; - tensor var_1487 = mul(x = qk_9, y = M_9)[name = tensor("op_1487")]; + tensor var_1269 = const()[name = tensor("op_1269"), val = tensor([1, 2])]; + tensor var_1270 = reshape(shape = var_1269, x = valid_mask)[name = tensor("op_1270")]; + tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1270)[name = tensor("causal_with_valid_1")]; + tensor var_1272 = const()[name = tensor("op_1272"), val = tensor([2, 1])]; + tensor var_1273 = reshape(shape = var_1272, x = sqrt_s_t_9)[name = tensor("op_1273")]; + tensor M_9 = real_div(x = causal_with_valid_1, y = var_1273)[name = tensor("M_9")]; + tensor var_1275 = mul(x = qk_9, y = M_9)[name = tensor("op_1275")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_1469)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1487, y = v_9)[name = tensor("inner_9")]; - tensor var_1489_transpose_x_0 = const()[name = tensor("op_1489_transpose_x_0"), val = tensor(false)]; - tensor var_1489_transpose_y_0 = const()[name = tensor("op_1489_transpose_y_0"), val = tensor(false)]; - tensor var_1489 = matmul(transpose_x = var_1489_transpose_x_0, transpose_y = var_1489_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1489")]; - tensor var_1490 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1490")]; - tensor var_1491 = const()[name = tensor("op_1491"), val = tensor([1, 1, 2, 1])]; - tensor var_1492 = reshape(shape = var_1491, x = var_1490)[name = tensor("op_1492")]; - tensor cross_9 = mul(x = var_1489, y = var_1492)[name = tensor("cross_9")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1257)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1275, y = v_9)[name = tensor("inner_9")]; + tensor var_1277_transpose_x_0 = const()[name = tensor("op_1277_transpose_x_0"), val = tensor(false)]; + tensor var_1277_transpose_y_0 = const()[name = tensor("op_1277_transpose_y_0"), val = tensor(false)]; + tensor var_1277 = matmul(transpose_x = var_1277_transpose_x_0, transpose_y = var_1277_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1277")]; + tensor var_1278 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1278")]; + tensor var_1279 = const()[name = tensor("op_1279"), val = tensor([1, 1, 2, 1])]; + tensor var_1280 = reshape(shape = var_1279, x = var_1278)[name = tensor("op_1280")]; + tensor cross_9 = mul(x = var_1277, y = var_1280)[name = tensor("cross_9")]; tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; - tensor var_1495 = const()[name = tensor("op_1495"), val = tensor([1, 1, 2, 1])]; - tensor var_1496 = reshape(shape = var_1495, x = valid_mask)[name = tensor("op_1496")]; - tensor v_masked_1 = mul(x = v_9, y = var_1496)[name = tensor("v_masked_1")]; - tensor var_1498 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1498")]; - tensor var_1500_transpose_x_1 = const()[name = tensor("op_1500_transpose_x_1"), val = tensor(true)]; - tensor var_1500_transpose_y_1 = const()[name = tensor("op_1500_transpose_y_1"), val = tensor(false)]; - tensor var_1500 = matmul(transpose_x = var_1500_transpose_x_1, transpose_y = var_1500_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1500")]; - tensor new_kv_unnorm_9 = add(x = var_1498, y = var_1500)[name = tensor("new_kv_unnorm_9")]; - tensor var_1502_keep_dims_0 = const()[name = tensor("op_1502_keep_dims_0"), val = tensor(false)]; - tensor var_1502 = reduce_sum(keep_dims = var_1502_keep_dims_0, x = valid_mask)[name = tensor("op_1502")]; - tensor var_1503 = const()[name = tensor("op_1503"), val = tensor([1])]; - tensor var_1504 = reshape(shape = var_1503, x = var_1502)[name = tensor("op_1504")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_1504)[name = tensor("new_scale_9")]; - tensor const_25 = const()[name = tensor("const_25"), val = tensor(0x1.fffffep+127)]; - tensor clip_18 = clip(alpha = var_1362, beta = const_25, x = new_scale_9)[name = tensor("clip_18")]; + tensor var_1283 = const()[name = tensor("op_1283"), val = tensor([1, 1, 2, 1])]; + tensor var_1284 = reshape(shape = var_1283, x = valid_mask)[name = tensor("op_1284")]; + tensor v_masked_1 = mul(x = v_9, y = var_1284)[name = tensor("v_masked_1")]; + tensor var_1286 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1286")]; + tensor var_1288_transpose_x_1 = const()[name = tensor("op_1288_transpose_x_1"), val = tensor(true)]; + tensor var_1288_transpose_y_1 = const()[name = tensor("op_1288_transpose_y_1"), val = tensor(false)]; + tensor var_1288 = matmul(transpose_x = var_1288_transpose_x_1, transpose_y = var_1288_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1288")]; + tensor new_kv_unnorm_9 = add(x = var_1286, y = var_1288)[name = tensor("new_kv_unnorm_9")]; + tensor var_1290_keep_dims_0 = const()[name = tensor("op_1290_keep_dims_0"), val = tensor(false)]; + tensor var_1290 = reduce_sum(keep_dims = var_1290_keep_dims_0, x = valid_mask)[name = tensor("op_1290")]; + tensor var_1291 = const()[name = tensor("op_1291"), val = tensor([1])]; + tensor var_1292 = reshape(shape = var_1291, x = var_1290)[name = tensor("op_1292")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1292)[name = tensor("new_scale_9")]; + tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; + tensor clip_18 = clip(alpha = var_1149, beta = const_21, x = new_scale_9)[name = tensor("clip_18")]; tensor sqrt_new_scale_1 = sqrt(x = clip_18)[name = tensor("sqrt_new_scale_1")]; - tensor var_1508 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1508")]; - tensor var_1509_perm_0 = const()[name = tensor("op_1509_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1296 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1296")]; + tensor var_1297_perm_0 = const()[name = tensor("op_1297_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_1509 = transpose(perm = var_1509_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1359, x = var_1509)[name = tensor("out_27")]; - tensor var_1513 = const()[name = tensor("op_1513"), val = tensor([12, 2, 256])]; - tensor out_29 = reshape(shape = var_1513, x = out_27)[name = tensor("out_29")]; - tensor var_1515 = silu(x = input_209)[name = tensor("op_1515")]; - tensor input_211 = mul(x = var_1515, y = out_29)[name = tensor("input_211")]; - tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_211)[name = tensor("linear_42")]; - tensor input_213 = add(x = x_37, y = ret_out_9)[name = tensor("input_213")]; + tensor var_1297 = transpose(perm = var_1297_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1146, x = var_1297)[name = tensor("out_27")]; + tensor var_1301 = const()[name = tensor("op_1301"), val = tensor([12, 2, 256])]; + tensor out_29 = reshape(shape = var_1301, x = out_27)[name = tensor("out_29")]; + tensor var_1303 = silu(x = input_169)[name = tensor("op_1303")]; + tensor input_171 = mul(x = var_1303, y = out_29)[name = tensor("input_171")]; + tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; + tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1357, gamma = decoder_norm11_0_weight, x = input_213)[name = tensor("xt_1")]; - tensor var_1525 = const()[name = tensor("op_1525"), val = tensor([1, 12, 2, 256])]; - tensor var_1526 = reshape(shape = var_1525, x = xt_1)[name = tensor("op_1526")]; - tensor var_1527_perm_0 = const()[name = tensor("op_1527_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1530 = const()[name = tensor("op_1530"), val = tensor([2, 12, 256])]; - tensor var_1527 = transpose(perm = var_1527_perm_0, x = var_1526)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_1530, x = var_1527)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1144, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1313 = const()[name = tensor("op_1313"), val = tensor([1, 12, 2, 256])]; + tensor var_1314 = reshape(shape = var_1313, x = xt_1)[name = tensor("op_1314")]; + tensor var_1315_perm_0 = const()[name = tensor("op_1315_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1318 = const()[name = tensor("op_1318"), val = tensor([2, 12, 256])]; + tensor var_1315 = transpose(perm = var_1315_perm_0, x = var_1314)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1318, x = var_1315)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_1553 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_8 = const()[name = tensor("concat_8"), val = tensor([12, 2, 3, 256])]; - tensor var_1555 = reshape(shape = concat_8, x = var_1553)[name = tensor("op_1555")]; - tensor var_1556_axes_0 = const()[name = tensor("op_1556_axes_0"), val = tensor([0])]; - tensor var_1556 = expand_dims(axes = var_1556_axes_0, x = var_1555)[name = tensor("op_1556")]; - tensor var_1557_perm_0 = const()[name = tensor("op_1557_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1558_axes_0 = const()[name = tensor("op_1558_axes_0"), val = tensor([-2])]; - tensor var_1557 = transpose(perm = var_1557_perm_0, x = var_1556)[name = tensor("transpose_21")]; - tensor var_1558 = squeeze(axes = var_1558_axes_0, x = var_1557)[name = tensor("op_1558")]; + tensor var_1341 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_9 = const()[name = tensor("concat_9"), val = tensor([12, 2, 3, 256])]; + tensor var_1343 = reshape(shape = concat_9, x = var_1341)[name = tensor("op_1343")]; + tensor var_1344_axes_0 = const()[name = tensor("op_1344_axes_0"), val = tensor([0])]; + tensor var_1344 = expand_dims(axes = var_1344_axes_0, x = var_1343)[name = tensor("op_1344")]; + tensor var_1345_perm_0 = const()[name = tensor("op_1345_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1346_axes_0 = const()[name = tensor("op_1346_axes_0"), val = tensor([-2])]; + tensor var_1345 = transpose(perm = var_1345_perm_0, x = var_1344)[name = tensor("transpose_21")]; + tensor var_1346 = squeeze(axes = var_1346_axes_0, x = var_1345)[name = tensor("op_1346")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 12, 2, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1558)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1346)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 12, 2, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1558)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1346)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 12, 2, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1558)[name = tensor("v_11")]; - tensor var_1566 = const()[name = tensor("op_1566"), val = tensor([12, 8, 64])]; - tensor var_1567 = reshape(shape = var_1566, x = q_11)[name = tensor("op_1567")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1346)[name = tensor("v_11")]; + tensor var_1354 = const()[name = tensor("op_1354"), val = tensor([12, 8, 64])]; + tensor var_1355 = reshape(shape = var_1354, x = q_11)[name = tensor("op_1355")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1573 = const()[name = tensor("op_1573"), val = tensor([12, 8, 64])]; - tensor var_1574 = reshape(shape = var_1573, x = k_11)[name = tensor("op_1574")]; + tensor var_1361 = const()[name = tensor("op_1361"), val = tensor([12, 8, 64])]; + tensor var_1362 = reshape(shape = var_1361, x = k_11)[name = tensor("op_1362")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1580 = const()[name = tensor("op_1580"), val = tensor([12, 8, 64])]; - tensor var_1581 = reshape(shape = var_1580, x = v_11)[name = tensor("op_1581")]; + tensor var_1368 = const()[name = tensor("op_1368"), val = tensor([12, 8, 64])]; + tensor var_1369 = reshape(shape = var_1368, x = v_11)[name = tensor("op_1369")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1584 = const()[name = tensor("op_1584"), val = tensor([2, 4, 12, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_1567)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_1584, x = q_13)[name = tensor("q_15")]; - tensor var_1586 = const()[name = tensor("op_1586"), val = tensor([2, 4, 12, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_1574)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_1586, x = k_13)[name = tensor("k_15")]; - tensor var_1588 = const()[name = tensor("op_1588"), val = tensor([2, 4, 12, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_1581)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_1588, x = v_13)[name = tensor("v_15")]; + tensor var_1372 = const()[name = tensor("op_1372"), val = tensor([2, 4, 12, 64])]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1355)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1372, x = q_13)[name = tensor("q_15")]; + tensor var_1374 = const()[name = tensor("op_1374"), val = tensor([2, 4, 12, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1362)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1374, x = k_13)[name = tensor("k_15")]; + tensor var_1376 = const()[name = tensor("op_1376"), val = tensor([2, 4, 12, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1369)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1376, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -1405,30 +1258,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_1591 = const()[name = tensor("op_1591"), val = tensor([2, 0, 1, 3])]; - tensor var_1596 = const()[name = tensor("op_1596"), val = tensor([24, 256])]; - tensor var_1592 = transpose(perm = var_1591, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_1596, x = var_1592)[name = tensor("attn_output_3")]; + tensor var_1379 = const()[name = tensor("op_1379"), val = tensor([2, 0, 1, 3])]; + tensor var_1384 = const()[name = tensor("op_1384"), val = tensor([24, 256])]; + tensor var_1380 = transpose(perm = var_1379, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1384, x = var_1380)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_1600 = const()[name = tensor("op_1600"), val = tensor([12, 2, 256])]; - tensor attn_output_7 = reshape(shape = var_1600, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1388 = const()[name = tensor("op_1388"), val = tensor([12, 2, 256])]; + tensor attn_output_7 = reshape(shape = var_1388, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; - tensor input_215 = add(x = query_1, y = sa2_out_1)[name = tensor("input_215")]; - tensor input_217_axes_0 = const()[name = tensor("input_217_axes_0"), val = tensor([-1])]; - tensor input_217 = layer_norm(axes = input_217_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1357, gamma = decoder_norm21_0_weight, x = input_215)[name = tensor("input_217")]; - tensor input_219 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_217)[name = tensor("linear_45")]; - tensor input_221 = relu(x = input_219)[name = tensor("input_221")]; - tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_221)[name = tensor("linear_46")]; - tensor input_223 = add(x = input_217, y = ffn_out_1)[name = tensor("input_223")]; + tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; + tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1144, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; + tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; + tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; + tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1357, gamma = decoder_norm22_0_weight, x = input_223)[name = tensor("xt_3")]; - tensor var_1620 = const()[name = tensor("op_1620"), val = tensor([1, 2, 12, 256])]; - tensor x_39 = reshape(shape = var_1620, x = xt_3)[name = tensor("x_39")]; - tensor var_1622_perm_0 = const()[name = tensor("op_1622_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1626 = const()[name = tensor("op_1626"), val = tensor([12, 2, 256])]; - tensor var_1622 = transpose(perm = var_1622_perm_0, x = x_39)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_1626, x = var_1622)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1144, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1408 = const()[name = tensor("op_1408"), val = tensor([1, 2, 12, 256])]; + tensor x_31 = reshape(shape = var_1408, x = xt_3)[name = tensor("x_31")]; + tensor var_1410_perm_0 = const()[name = tensor("op_1410_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1414 = const()[name = tensor("op_1414"), val = tensor([12, 2, 256])]; + tensor var_1410 = transpose(perm = var_1410_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1414, x = var_1410)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 12, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1439,120 +1292,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_1634 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_1635 = const()[name = tensor("op_1635"), val = tensor([12, 2, 4, 64])]; - tensor var_1636 = reshape(shape = var_1635, x = var_1634)[name = tensor("op_1636")]; + tensor var_1422 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1423 = const()[name = tensor("op_1423"), val = tensor([12, 2, 4, 64])]; + tensor var_1424 = reshape(shape = var_1423, x = var_1422)[name = tensor("op_1424")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1640 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_1641 = const()[name = tensor("op_1641"), val = tensor(0x1p-3)]; - tensor var_1642 = mul(x = var_1640, y = var_1641)[name = tensor("op_1642")]; - tensor var_1643 = const()[name = tensor("op_1643"), val = tensor([12, 2, 4, 64])]; - tensor var_1644 = reshape(shape = var_1643, x = var_1642)[name = tensor("op_1644")]; + tensor var_1428 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1429 = const()[name = tensor("op_1429"), val = tensor(0x1p-3)]; + tensor var_1430 = mul(x = var_1428, y = var_1429)[name = tensor("op_1430")]; + tensor var_1431 = const()[name = tensor("op_1431"), val = tensor([12, 2, 4, 64])]; + tensor var_1432 = reshape(shape = var_1431, x = var_1430)[name = tensor("op_1432")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1648 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_1649 = const()[name = tensor("op_1649"), val = tensor([12, 2, 4, 64])]; - tensor var_1650 = reshape(shape = var_1649, x = var_1648)[name = tensor("op_1650")]; + tensor var_1436 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1437 = const()[name = tensor("op_1437"), val = tensor([12, 2, 4, 64])]; + tensor var_1438 = reshape(shape = var_1437, x = var_1436)[name = tensor("op_1438")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_227 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; + tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; - tensor const_36 = const()[name = tensor("const_36"), val = tensor(0x1.fffffep+127)]; - tensor clip_19 = clip(alpha = var_1362, beta = const_36, x = s_t)[name = tensor("clip_19")]; + tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; + tensor clip_19 = clip(alpha = var_1149, beta = const_32, x = s_t)[name = tensor("clip_19")]; tensor sqrt_s_t = sqrt(x = clip_19)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_1644)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_1636)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1432)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1424)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_1665 = const()[name = tensor("op_1665"), val = tensor([2, 1])]; - tensor var_1666 = reshape(shape = var_1665, x = sqrt_s_t)[name = tensor("op_1666")]; - tensor M = real_div(x = causal_with_valid_1, y = var_1666)[name = tensor("M")]; - tensor var_1668 = mul(x = qk, y = M)[name = tensor("op_1668")]; + tensor var_1453 = const()[name = tensor("op_1453"), val = tensor([2, 1])]; + tensor var_1454 = reshape(shape = var_1453, x = sqrt_s_t)[name = tensor("op_1454")]; + tensor M = real_div(x = causal_with_valid_1, y = var_1454)[name = tensor("M")]; + tensor var_1456 = mul(x = qk, y = M)[name = tensor("op_1456")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_1650)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1668, y = v_17)[name = tensor("inner")]; - tensor var_1670_transpose_x_0 = const()[name = tensor("op_1670_transpose_x_0"), val = tensor(false)]; - tensor var_1670_transpose_y_0 = const()[name = tensor("op_1670_transpose_y_0"), val = tensor(false)]; - tensor var_1670 = matmul(transpose_x = var_1670_transpose_x_0, transpose_y = var_1670_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1670")]; - tensor var_1671 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1671")]; - tensor var_1672 = const()[name = tensor("op_1672"), val = tensor([1, 1, 2, 1])]; - tensor var_1673 = reshape(shape = var_1672, x = var_1671)[name = tensor("op_1673")]; - tensor cross = mul(x = var_1670, y = var_1673)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1438)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1456, y = v_17)[name = tensor("inner")]; + tensor var_1458_transpose_x_0 = const()[name = tensor("op_1458_transpose_x_0"), val = tensor(false)]; + tensor var_1458_transpose_y_0 = const()[name = tensor("op_1458_transpose_y_0"), val = tensor(false)]; + tensor var_1458 = matmul(transpose_x = var_1458_transpose_x_0, transpose_y = var_1458_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1458")]; + tensor var_1459 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1459")]; + tensor var_1460 = const()[name = tensor("op_1460"), val = tensor([1, 1, 2, 1])]; + tensor var_1461 = reshape(shape = var_1460, x = var_1459)[name = tensor("op_1461")]; + tensor cross = mul(x = var_1458, y = var_1461)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_1496)[name = tensor("v_masked")]; - tensor var_1679 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1679")]; - tensor var_1681_transpose_x_1 = const()[name = tensor("op_1681_transpose_x_1"), val = tensor(true)]; - tensor var_1681_transpose_y_1 = const()[name = tensor("op_1681_transpose_y_1"), val = tensor(false)]; - tensor var_1681 = matmul(transpose_x = var_1681_transpose_x_1, transpose_y = var_1681_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1681")]; - tensor new_kv_unnorm = add(x = var_1679, y = var_1681)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_1504)[name = tensor("new_scale")]; - tensor const_37 = const()[name = tensor("const_37"), val = tensor(0x1.fffffep+127)]; - tensor clip_20 = clip(alpha = var_1362, beta = const_37, x = new_scale)[name = tensor("clip_20")]; + tensor v_masked = mul(x = v_17, y = var_1284)[name = tensor("v_masked")]; + tensor var_1467 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1467")]; + tensor var_1469_transpose_x_1 = const()[name = tensor("op_1469_transpose_x_1"), val = tensor(true)]; + tensor var_1469_transpose_y_1 = const()[name = tensor("op_1469_transpose_y_1"), val = tensor(false)]; + tensor var_1469 = matmul(transpose_x = var_1469_transpose_x_1, transpose_y = var_1469_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1469")]; + tensor new_kv_unnorm = add(x = var_1467, y = var_1469)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1292)[name = tensor("new_scale")]; + tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; + tensor clip_20 = clip(alpha = var_1149, beta = const_33, x = new_scale)[name = tensor("clip_20")]; tensor sqrt_new_scale = sqrt(x = clip_20)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_1690_perm_0 = const()[name = tensor("op_1690_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1478_perm_0 = const()[name = tensor("op_1478_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_1690 = transpose(perm = var_1690_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1359, x = var_1690)[name = tensor("out_33")]; - tensor var_1694 = const()[name = tensor("op_1694"), val = tensor([12, 2, 256])]; - tensor out = reshape(shape = var_1694, x = out_33)[name = tensor("out")]; - tensor var_1696 = silu(x = input_227)[name = tensor("op_1696")]; - tensor input_229 = mul(x = var_1696, y = out)[name = tensor("input_229")]; - tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_229)[name = tensor("linear_51")]; - tensor input_231 = add(x = x, y = ret_out)[name = tensor("input_231")]; + tensor var_1478 = transpose(perm = var_1478_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1146, x = var_1478)[name = tensor("out_33")]; + tensor var_1482 = const()[name = tensor("op_1482"), val = tensor([12, 2, 256])]; + tensor out = reshape(shape = var_1482, x = out_33)[name = tensor("out")]; + tensor var_1484 = silu(x = input_187)[name = tensor("op_1484")]; + tensor input_189 = mul(x = var_1484, y = out)[name = tensor("input_189")]; + tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; + tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1357, gamma = decoder_norm11_1_weight, x = input_231)[name = tensor("xt_5")]; - tensor var_1706 = const()[name = tensor("op_1706"), val = tensor([1, 12, 2, 256])]; - tensor var_1707 = reshape(shape = var_1706, x = xt_5)[name = tensor("op_1707")]; - tensor var_1708_perm_0 = const()[name = tensor("op_1708_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1711 = const()[name = tensor("op_1711"), val = tensor([2, 12, 256])]; - tensor var_1708 = transpose(perm = var_1708_perm_0, x = var_1707)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_1711, x = var_1708)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1144, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1494 = const()[name = tensor("op_1494"), val = tensor([1, 12, 2, 256])]; + tensor var_1495 = reshape(shape = var_1494, x = xt_5)[name = tensor("op_1495")]; + tensor var_1496_perm_0 = const()[name = tensor("op_1496_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1499 = const()[name = tensor("op_1499"), val = tensor([2, 12, 256])]; + tensor var_1496 = transpose(perm = var_1496_perm_0, x = var_1495)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1499, x = var_1496)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_1734 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_9 = const()[name = tensor("concat_9"), val = tensor([12, 2, 3, 256])]; - tensor var_1736 = reshape(shape = concat_9, x = var_1734)[name = tensor("op_1736")]; - tensor var_1737_axes_0 = const()[name = tensor("op_1737_axes_0"), val = tensor([0])]; - tensor var_1737 = expand_dims(axes = var_1737_axes_0, x = var_1736)[name = tensor("op_1737")]; - tensor var_1738_perm_0 = const()[name = tensor("op_1738_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1739_axes_0 = const()[name = tensor("op_1739_axes_0"), val = tensor([-2])]; - tensor var_1738 = transpose(perm = var_1738_perm_0, x = var_1737)[name = tensor("transpose_8")]; - tensor var_1739 = squeeze(axes = var_1739_axes_0, x = var_1738)[name = tensor("op_1739")]; + tensor var_1522 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_10 = const()[name = tensor("concat_10"), val = tensor([12, 2, 3, 256])]; + tensor var_1524 = reshape(shape = concat_10, x = var_1522)[name = tensor("op_1524")]; + tensor var_1525_axes_0 = const()[name = tensor("op_1525_axes_0"), val = tensor([0])]; + tensor var_1525 = expand_dims(axes = var_1525_axes_0, x = var_1524)[name = tensor("op_1525")]; + tensor var_1526_perm_0 = const()[name = tensor("op_1526_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1527_axes_0 = const()[name = tensor("op_1527_axes_0"), val = tensor([-2])]; + tensor var_1526 = transpose(perm = var_1526_perm_0, x = var_1525)[name = tensor("transpose_8")]; + tensor var_1527 = squeeze(axes = var_1527_axes_0, x = var_1526)[name = tensor("op_1527")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 12, 2, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1739)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1527)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 12, 2, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1739)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1527)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 12, 2, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1739)[name = tensor("v_19")]; - tensor var_1747 = const()[name = tensor("op_1747"), val = tensor([12, 8, 64])]; - tensor var_1748 = reshape(shape = var_1747, x = q_19)[name = tensor("op_1748")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1527)[name = tensor("v_19")]; + tensor var_1535 = const()[name = tensor("op_1535"), val = tensor([12, 8, 64])]; + tensor var_1536 = reshape(shape = var_1535, x = q_19)[name = tensor("op_1536")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1754 = const()[name = tensor("op_1754"), val = tensor([12, 8, 64])]; - tensor var_1755 = reshape(shape = var_1754, x = k_19)[name = tensor("op_1755")]; + tensor var_1542 = const()[name = tensor("op_1542"), val = tensor([12, 8, 64])]; + tensor var_1543 = reshape(shape = var_1542, x = k_19)[name = tensor("op_1543")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1761 = const()[name = tensor("op_1761"), val = tensor([12, 8, 64])]; - tensor var_1762 = reshape(shape = var_1761, x = v_19)[name = tensor("op_1762")]; + tensor var_1549 = const()[name = tensor("op_1549"), val = tensor([12, 8, 64])]; + tensor var_1550 = reshape(shape = var_1549, x = v_19)[name = tensor("op_1550")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1765 = const()[name = tensor("op_1765"), val = tensor([2, 4, 12, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_1748)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_1765, x = q_21)[name = tensor("q")]; - tensor var_1767 = const()[name = tensor("op_1767"), val = tensor([2, 4, 12, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_1755)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_1767, x = k_21)[name = tensor("k")]; - tensor var_1769 = const()[name = tensor("op_1769"), val = tensor([2, 4, 12, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_1762)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_1769, x = v_21)[name = tensor("v")]; + tensor var_1553 = const()[name = tensor("op_1553"), val = tensor([2, 4, 12, 64])]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1536)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1553, x = q_21)[name = tensor("q")]; + tensor var_1555 = const()[name = tensor("op_1555"), val = tensor([2, 4, 12, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1543)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1555, x = k_21)[name = tensor("k")]; + tensor var_1557 = const()[name = tensor("op_1557"), val = tensor([2, 4, 12, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1550)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1557, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -1563,50 +1416,50 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_1772 = const()[name = tensor("op_1772"), val = tensor([2, 0, 1, 3])]; - tensor var_1777 = const()[name = tensor("op_1777"), val = tensor([24, 256])]; - tensor var_1773 = transpose(perm = var_1772, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_1777, x = var_1773)[name = tensor("attn_output_11")]; + tensor var_1560 = const()[name = tensor("op_1560"), val = tensor([2, 0, 1, 3])]; + tensor var_1565 = const()[name = tensor("op_1565"), val = tensor([24, 256])]; + tensor var_1561 = transpose(perm = var_1560, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1565, x = var_1561)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_1781 = const()[name = tensor("op_1781"), val = tensor([12, 2, 256])]; - tensor attn_output = reshape(shape = var_1781, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1569 = const()[name = tensor("op_1569"), val = tensor([12, 2, 256])]; + tensor attn_output = reshape(shape = var_1569, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; - tensor input_233 = add(x = query_5, y = sa2_out)[name = tensor("input_233")]; - tensor input_235_axes_0 = const()[name = tensor("input_235_axes_0"), val = tensor([-1])]; - tensor input_235 = layer_norm(axes = input_235_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1357, gamma = decoder_norm21_1_weight, x = input_233)[name = tensor("input_235")]; - tensor input_237 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_235)[name = tensor("linear_54")]; - tensor input_239 = relu(x = input_237)[name = tensor("input_239")]; - tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_239)[name = tensor("linear_55")]; - tensor input_241 = add(x = input_235, y = ffn_out)[name = tensor("input_241")]; + tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; + tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1144, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; + tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; + tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; + tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1357, gamma = decoder_norm22_1_weight, x = input_241)[name = tensor("xt")]; - tensor var_1801 = const()[name = tensor("op_1801"), val = tensor([1, 2, 12, 256])]; - tensor input = reshape(shape = var_1801, x = xt)[name = tensor("input")]; - tensor var_1803 = const()[name = tensor("op_1803"), val = tensor([-1])]; - tensor var_1804 = reduce_l2_norm(axes = var_1803, keep_dims = var_1360, x = input)[name = tensor("op_1804")]; - tensor const_46 = const()[name = tensor("const_46"), val = tensor(0x1.fffffep+127)]; - tensor clip_21 = clip(alpha = var_1351, beta = const_46, x = var_1804)[name = tensor("clip_21")]; - tensor var_1806 = real_div(x = input, y = clip_21)[name = tensor("op_1806")]; - tensor concat_13 = const()[name = tensor("concat_13"), val = tensor([2, 1, 256])]; - tensor reshape_0 = reshape(shape = concat_13, x = emb)[name = tensor("reshape_0")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1144, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1589 = const()[name = tensor("op_1589"), val = tensor([1, 2, 12, 256])]; + tensor input = reshape(shape = var_1589, x = xt)[name = tensor("input")]; + tensor var_1591 = const()[name = tensor("op_1591"), val = tensor([-1])]; + tensor var_1592 = reduce_l2_norm(axes = var_1591, keep_dims = var_1147, x = input)[name = tensor("op_1592")]; + tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; + tensor clip_21 = clip(alpha = var_1139, beta = const_42, x = var_1592)[name = tensor("clip_21")]; + tensor var_1594 = real_div(x = input, y = clip_21)[name = tensor("op_1594")]; + tensor concat_14 = const()[name = tensor("concat_14"), val = tensor([2, 1, 256])]; + tensor reshape_0 = reshape(shape = concat_14, x = emb)[name = tensor("reshape_0")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_14 = const()[name = tensor("concat_14"), val = tensor([2, 256, 12])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1806)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_14, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_15 = const()[name = tensor("concat_15"), val = tensor([2, 256, 12])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1594)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_15, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = reshape_0, y = reshape_1)[name = tensor("matmul_2")]; - tensor concat_17 = const()[name = tensor("concat_17"), val = tensor([1, 2, 12])]; - tensor reshape_2 = reshape(shape = concat_17, x = matmul_2)[name = tensor("reshape_2")]; + tensor concat_18 = const()[name = tensor("concat_18"), val = tensor([1, 2, 12])]; + tensor reshape_2 = reshape(shape = concat_18, x = matmul_2)[name = tensor("reshape_2")]; tensor output_begin_0 = const()[name = tensor("output_begin_0"), val = tensor([0, 0, 1])]; tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 2, 11])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = reshape_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_1810")]; - tensor var_1812_axis_0 = const()[name = tensor("op_1812_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_1812_axis_0, values = (var_1508, nkv))[name = tensor("op_1812")]; - tensor var_1814_axis_0 = const()[name = tensor("op_1814_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_1814_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1814")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1598")]; + tensor var_1600_axis_0 = const()[name = tensor("op_1600_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1600_axis_0, values = (var_1296, nkv))[name = tensor("op_1600")]; + tensor var_1602_axis_0 = const()[name = tensor("op_1602_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1602_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1602")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/DIHARD III/ls_eend_dih3_200ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/DIHARD III/ls_eend_dih3_200ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index 35c212ad75f701c18961c9f0a8fafb1431326688..eb69de5e7b602464939d1d7ce343db9d96e54074 100644 --- a/optimized/DIHARD III/ls_eend_dih3_200ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/DIHARD III/ls_eend_dih3_200ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:23eaac3846ae5656304318bb920b7cb2399a66ca91519d271cd841277b639c58 -size 228675 +oid sha256:72f6e64fc4b57055c77cad85ce273db4e89d0442730e221cff703d1e39713b75 +size 205526 diff --git a/optimized/DIHARD III/ls_eend_dih3_200ms.mlpackage/Manifest.json b/optimized/DIHARD III/ls_eend_dih3_200ms.mlpackage/Manifest.json index e4adf35fdf52250b44ac16b93dd352216d1a2a46..4edd47baf7d9cc56d706652b3e39e50e3af58fbc 100644 --- a/optimized/DIHARD III/ls_eend_dih3_200ms.mlpackage/Manifest.json +++ b/optimized/DIHARD III/ls_eend_dih3_200ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "A2777964-3446-429E-B6F6-CC8CCB530674": { - "author": "com.apple.CoreML", - "description": "CoreML Model Weights", - "name": "weights", - "path": "com.apple.CoreML/weights" - }, - "C649B447-4D42-417C-96B6-A60F20D58FB4": { + "1EB451EE-795E-4BC6-9609-BEA34EB9A6C9": { "author": "com.apple.CoreML", "description": "CoreML Model Specification", "name": "model.mlmodel", "path": "com.apple.CoreML/model.mlmodel" + }, + "C152DC14-ECF4-4693-922C-6C5AD6A2ACC7": { + "author": "com.apple.CoreML", + "description": "CoreML Model Weights", + "name": "weights", + "path": "com.apple.CoreML/weights" } }, - "rootModelIdentifier": "C649B447-4D42-417C-96B6-A60F20D58FB4" + "rootModelIdentifier": "1EB451EE-795E-4BC6-9609-BEA34EB9A6C9" } diff --git a/optimized/DIHARD III/ls_eend_dih3_300ms.mlmodelc/analytics/coremldata.bin b/optimized/DIHARD III/ls_eend_dih3_300ms.mlmodelc/analytics/coremldata.bin index 73b5e54d07cff02f50469b50daec4e3fc89464ea..35a5d5348ec4cb2e4284290f9b3e4a200c9f28f7 100644 --- a/optimized/DIHARD III/ls_eend_dih3_300ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/DIHARD III/ls_eend_dih3_300ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3818b2f701ebde88a9728159c936a5c8342c93ae0eeaecb9a0dcb1e8dd6f7335 +oid sha256:9c742800780790f4577eec1bc7e0caf6560020d2cbee5852bb35a9296b88df31 size 243 diff --git a/optimized/DIHARD III/ls_eend_dih3_300ms.mlmodelc/coremldata.bin b/optimized/DIHARD III/ls_eend_dih3_300ms.mlmodelc/coremldata.bin index d893ad6a91a9602e0e0a1e840064849997afd2c4..bded976ee7d04319d8c90c858d0218b5ff94f5c7 100644 --- a/optimized/DIHARD III/ls_eend_dih3_300ms.mlmodelc/coremldata.bin +++ b/optimized/DIHARD III/ls_eend_dih3_300ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a789fc19ef67d26befa8c8282d69f102970c9f1bdfb14fe1334fa43b7c08ca47 -size 1268 +oid sha256:359290d39bd3df95180c594e376d506fd2b060be232db339a55da2f3992d1f91 +size 1359 diff --git a/optimized/DIHARD III/ls_eend_dih3_300ms.mlmodelc/metadata.json b/optimized/DIHARD III/ls_eend_dih3_300ms.mlmodelc/metadata.json index 9eb7a7c0b9a5485e9aa7087b29536e2dad80ba75..37f70c3b595ccbf6b3bf57c6eca167a30e6cb6d4 100644 --- a/optimized/DIHARD III/ls_eend_dih3_300ms.mlmodelc/metadata.json +++ b/optimized/DIHARD III/ls_eend_dih3_300ms.mlmodelc/metadata.json @@ -96,30 +96,30 @@ "Ios17.reshape" : 92, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 69, - "Split" : 12, + "Ios17.transpose" : 57, + "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 82, "Tile" : 13, - "Ios17.sliceByIndex" : 76, - "Ios16.sigmoid" : 13, + "Ios17.sliceByIndex" : 60, + "Ios16.sigmoid" : 5, "Ios16.reduceSum" : 1, "Ios17.squeeze" : 2, - "Ios17.layerNorm" : 41, - "Ios17.batchNorm" : 12, + "Ios17.layerNorm" : 33, + "Ios17.batchNorm" : 4, "Ios17.cast" : 24, "Ios17.sqrt" : 18, "Ios17.sub" : 24, - "Ios17.conv" : 37, + "Ios17.conv" : 13, "Ios17.realDiv" : 20, "Ios17.linear" : 56, "Ios17.concat" : 18, "Ios17.clip" : 30, "Ios17.greaterEqual" : 12, - "Ios16.silu" : 26, + "Ios16.silu" : 18, "Stack" : 6, "Ios16.cumsum" : 1, - "Ios17.mul" : 103 + "Ios17.mul" : 95 }, "computePrecision" : "Mixed (Float32, Int32)", "isUpdatable" : "0", @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"dih3\", \"model_label\": \"DIHARD III\", \"variant\": \"pipeline\", \"T\": 3, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"dih3\", \"model_label\": \"DIHARD III\", \"variant\": \"pipeline\", \"chunk_size\": 3, \"step_duration_ms\": 300, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/DIHARD III/ls_eend_dih3_300ms.mlmodelc/model.mil b/optimized/DIHARD III/ls_eend_dih3_300ms.mlmodelc/model.mil index 4d8299dcfef93c6b52fa3dc309c7deb5a9f033c0..002a08ea435aee3ce73f649aad1d104b06bba03d 100644 --- a/optimized/DIHARD III/ls_eend_dih3_300ms.mlmodelc/model.mil +++ b/optimized/DIHARD III/ls_eend_dih3_300ms.mlmodelc/model.mil @@ -217,6 +217,7 @@ program(1.0) tensor var_21 = const()[name = tensor("op_21"), val = tensor(0x0p+0)]; tensor var_24 = const()[name = tensor("op_24"), val = tensor(0x1.0c6f7ap-20)]; tensor var_27 = const()[name = tensor("op_27"), val = tensor(2)]; + tensor var_30 = const()[name = tensor("op_30"), val = tensor(0)]; tensor var_33 = const()[name = tensor("op_33"), val = tensor(1)]; tensor var_35 = const()[name = tensor("op_35"), val = tensor(0x1.4f8b58p-17)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(true)]; @@ -263,8 +264,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_69")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_70")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_196 = const()[name = tensor("op_196"), val = tensor([3, 1])]; tensor var_197 = reshape(shape = var_196, x = sqrt_s_t_1)[name = tensor("op_197")]; @@ -272,7 +273,7 @@ program(1.0) tensor var_199 = mul(x = qk_1, y = M_1)[name = tensor("op_199")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_68")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_199, y = v_1)[name = tensor("inner_1")]; tensor var_201_transpose_x_0 = const()[name = tensor("op_201_transpose_x_0"), val = tensor(false)]; tensor var_201_transpose_y_0 = const()[name = tensor("op_201_transpose_y_0"), val = tensor(false)]; @@ -293,7 +294,7 @@ program(1.0) tensor var_214 = real_div(x = new_kv_unnorm_1, y = var_213)[name = tensor("op_214")]; tensor var_215_perm_0 = const()[name = tensor("op_215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_67")]; + tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_24, x = var_215)[name = tensor("out_3")]; tensor var_219 = const()[name = tensor("op_219"), val = tensor([1, 3, 256])]; tensor out_5 = reshape(shape = var_219, x = out_3)[name = tensor("out_5")]; @@ -341,218 +342,142 @@ program(1.0) tensor var_252 = sub(x = var_17, y = use_roll_1)[name = tensor("op_252")]; tensor var_253 = mul(x = filled_window_1, y = var_252)[name = tensor("op_253")]; tensor var_254 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_254")]; - tensor input_21 = add(x = var_253, y = var_254)[name = tensor("input_21")]; + tensor window_3 = add(x = var_253, y = var_254)[name = tensor("window_3")]; tensor var_256 = const()[name = tensor("op_256"), val = tensor(0x1p+0)]; tensor var_257 = add(x = filled_1, y = var_256)[name = tensor("op_257")]; tensor clip_1 = clip(alpha = var_21, beta = var_16, x = var_257)[name = tensor("clip_1")]; + tensor var_260_begin_0 = const()[name = tensor("op_260_begin_0"), val = tensor([0, 1, 0])]; + tensor var_260_end_0 = const()[name = tensor("op_260_end_0"), val = tensor([1, 2, 256])]; + tensor var_260_end_mask_0 = const()[name = tensor("op_260_end_mask_0"), val = tensor([true, false, true])]; + tensor var_260 = slice_by_index(begin = var_260_begin_0, end = var_260_end_0, end_mask = var_260_end_mask_0, x = x_3)[name = tensor("op_260")]; + tensor var_263_begin_0 = const()[name = tensor("op_263_begin_0"), val = tensor([0, 1, 0])]; + tensor var_263_end_0 = const()[name = tensor("op_263_end_0"), val = tensor([1, 16, 256])]; + tensor var_263_end_mask_0 = const()[name = tensor("op_263_end_mask_0"), val = tensor([true, true, true])]; + tensor var_263 = slice_by_index(begin = var_263_begin_0, end = var_263_end_0, end_mask = var_263_end_mask_0, x = window_3)[name = tensor("op_263")]; + tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; + tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_263, var_260))[name = tensor("rolled_3")]; + tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; + tensor var_268 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_268")]; + tensor var_269_dtype_0 = const()[name = tensor("op_269_dtype_0"), val = tensor("fp32")]; + tensor var_270 = const()[name = tensor("op_270"), val = tensor([1, 16, 1])]; + tensor var_269 = cast(dtype = var_269_dtype_0, x = var_268)[name = tensor("cast_157")]; + tensor fill_mask_3 = reshape(shape = var_270, x = var_269)[name = tensor("fill_mask_3")]; + tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_260)[name = tensor("x_expanded_3")]; + tensor var_274 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_274")]; + tensor var_275 = mul(x = window_3, y = var_274)[name = tensor("op_275")]; + tensor var_276 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_276")]; + tensor filled_window_3 = add(x = var_275, y = var_276)[name = tensor("filled_window_3")]; + tensor var_278 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_278")]; + tensor var_279_dtype_0 = const()[name = tensor("op_279_dtype_0"), val = tensor("fp32")]; + tensor var_280 = const()[name = tensor("op_280"), val = tensor([1, 1, 1])]; + tensor var_279 = cast(dtype = var_279_dtype_0, x = var_278)[name = tensor("cast_156")]; + tensor use_roll_3 = reshape(shape = var_280, x = var_279)[name = tensor("use_roll_3")]; + tensor var_282 = sub(x = var_17, y = use_roll_3)[name = tensor("op_282")]; + tensor var_283 = mul(x = filled_window_3, y = var_282)[name = tensor("op_283")]; + tensor var_284 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_284")]; + tensor window_5 = add(x = var_283, y = var_284)[name = tensor("window_5")]; + tensor var_286 = const()[name = tensor("op_286"), val = tensor(0x1p+0)]; + tensor var_287 = add(x = clip_1, y = var_286)[name = tensor("op_287")]; + tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_287)[name = tensor("clip_3")]; + tensor var_290_begin_0 = const()[name = tensor("op_290_begin_0"), val = tensor([0, 2, 0])]; + tensor var_290_end_0 = const()[name = tensor("op_290_end_0"), val = tensor([1, 1, 256])]; + tensor var_290_end_mask_0 = const()[name = tensor("op_290_end_mask_0"), val = tensor([true, true, true])]; + tensor var_290 = slice_by_index(begin = var_290_begin_0, end = var_290_end_0, end_mask = var_290_end_mask_0, x = x_3)[name = tensor("op_290")]; + tensor var_293_begin_0 = const()[name = tensor("op_293_begin_0"), val = tensor([0, 1, 0])]; + tensor var_293_end_0 = const()[name = tensor("op_293_end_0"), val = tensor([1, 16, 256])]; + tensor var_293_end_mask_0 = const()[name = tensor("op_293_end_mask_0"), val = tensor([true, true, true])]; + tensor var_293 = slice_by_index(begin = var_293_begin_0, end = var_293_end_0, end_mask = var_293_end_mask_0, x = window_5)[name = tensor("op_293")]; + tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; + tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_293, var_290))[name = tensor("rolled_5")]; + tensor clip_4 = clip(alpha = var_21, beta = var_20, x = clip_3)[name = tensor("clip_4")]; + tensor var_298 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_298")]; + tensor var_299_dtype_0 = const()[name = tensor("op_299_dtype_0"), val = tensor("fp32")]; + tensor var_300 = const()[name = tensor("op_300"), val = tensor([1, 16, 1])]; + tensor var_299 = cast(dtype = var_299_dtype_0, x = var_298)[name = tensor("cast_155")]; + tensor fill_mask_5 = reshape(shape = var_300, x = var_299)[name = tensor("fill_mask_5")]; + tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_290)[name = tensor("x_expanded_5")]; + tensor var_304 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_304")]; + tensor var_305 = mul(x = window_5, y = var_304)[name = tensor("op_305")]; + tensor var_306 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_306")]; + tensor filled_window_5 = add(x = var_305, y = var_306)[name = tensor("filled_window_5")]; + tensor var_308 = greater_equal(x = clip_3, y = var_16)[name = tensor("op_308")]; + tensor var_309_dtype_0 = const()[name = tensor("op_309_dtype_0"), val = tensor("fp32")]; + tensor var_310 = const()[name = tensor("op_310"), val = tensor([1, 1, 1])]; + tensor var_309 = cast(dtype = var_309_dtype_0, x = var_308)[name = tensor("cast_154")]; + tensor use_roll_5 = reshape(shape = var_310, x = var_309)[name = tensor("use_roll_5")]; + tensor var_312 = sub(x = var_17, y = use_roll_5)[name = tensor("op_312")]; + tensor var_313 = mul(x = filled_window_5, y = var_312)[name = tensor("op_313")]; + tensor var_314 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_314")]; + tensor window_7 = add(x = var_313, y = var_314)[name = tensor("window_7")]; + tensor var_316 = const()[name = tensor("op_316"), val = tensor(0x1p+0)]; + tensor var_317 = add(x = clip_3, y = var_316)[name = tensor("op_317")]; + tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_317)[name = tensor("clip_5")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_30, interleave = input_21_interleave_0, values = (window_3, window_5, window_7))[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; - tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; + tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; tensor inputs_3_pad_type_0 = const()[name = tensor("inputs_3_pad_type_0"), val = tensor("valid")]; tensor inputs_3_strides_0 = const()[name = tensor("inputs_3_strides_0"), val = tensor([1])]; tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_66")]; - tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_278_split_sizes_0 = const()[name = tensor("op_278_split_sizes_0"), val = tensor([256, 256])]; - tensor var_278_axis_0 = const()[name = tensor("op_278_axis_0"), val = tensor(1)]; - tensor var_278_0, tensor var_278_1 = split(axis = var_278_axis_0, split_sizes = var_278_split_sizes_0, x = inputs_3)[name = tensor("op_278")]; - tensor var_280 = sigmoid(x = var_278_1)[name = tensor("op_280")]; - tensor inputs_5 = mul(x = var_278_0, y = var_280)[name = tensor("inputs_5")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; + tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; + tensor var_340_split_sizes_0 = const()[name = tensor("op_340_split_sizes_0"), val = tensor([256, 256])]; + tensor var_340_axis_0 = const()[name = tensor("op_340_axis_0"), val = tensor(1)]; + tensor var_340_0, tensor var_340_1 = split(axis = var_340_axis_0, split_sizes = var_340_split_sizes_0, x = inputs_3)[name = tensor("op_340")]; + tensor var_342 = sigmoid(x = var_340_1)[name = tensor("op_342")]; + tensor inputs_5 = mul(x = var_340_0, y = var_342)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; tensor outputs_aug_1_strides_0 = const()[name = tensor("outputs_aug_1_strides_0"), val = tensor([1])]; tensor outputs_aug_1_dilations_0 = const()[name = tensor("outputs_aug_1_dilations_0"), val = tensor([1])]; - tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; + tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; tensor input_25_begin_0 = const()[name = tensor("input_25_begin_0"), val = tensor([0, 0, 0])]; - tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([1, 256, 16])]; + tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([3, 256, 16])]; tensor input_25_end_mask_0 = const()[name = tensor("input_25_end_mask_0"), val = tensor([true, true, false])]; - tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; - tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; - tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; + tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; + tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; + tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; tensor input_29_pad_type_0 = const()[name = tensor("input_29_pad_type_0"), val = tensor("valid")]; tensor input_29_strides_0 = const()[name = tensor("input_29_strides_0"), val = tensor([1])]; tensor input_29_pad_0 = const()[name = tensor("input_29_pad_0"), val = tensor([0, 0])]; tensor input_29_dilations_0 = const()[name = tensor("input_29_dilations_0"), val = tensor([1])]; tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; - tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; + tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_311_begin_0 = const()[name = tensor("op_311_begin_0"), val = tensor([0, -1, 0])]; - tensor var_311_end_0 = const()[name = tensor("op_311_end_0"), val = tensor([1, 16, 256])]; - tensor var_311_end_mask_0 = const()[name = tensor("op_311_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_65")]; - tensor var_311 = slice_by_index(begin = var_311_begin_0, end = var_311_end_0, end_mask = var_311_end_mask_0, x = conv_out_1)[name = tensor("op_311")]; - tensor var_314_begin_0 = const()[name = tensor("op_314_begin_0"), val = tensor([0, 1, 0])]; - tensor var_314_end_0 = const()[name = tensor("op_314_end_0"), val = tensor([1, 2, 256])]; - tensor var_314_end_mask_0 = const()[name = tensor("op_314_end_mask_0"), val = tensor([true, false, true])]; - tensor var_314 = slice_by_index(begin = var_314_begin_0, end = var_314_end_0, end_mask = var_314_end_mask_0, x = x_3)[name = tensor("op_314")]; - tensor var_317_begin_0 = const()[name = tensor("op_317_begin_0"), val = tensor([0, 1, 0])]; - tensor var_317_end_0 = const()[name = tensor("op_317_end_0"), val = tensor([1, 16, 256])]; - tensor var_317_end_mask_0 = const()[name = tensor("op_317_end_mask_0"), val = tensor([true, true, true])]; - tensor var_317 = slice_by_index(begin = var_317_begin_0, end = var_317_end_0, end_mask = var_317_end_mask_0, x = input_21)[name = tensor("op_317")]; - tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_317, var_314))[name = tensor("rolled_3")]; - tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; - tensor var_322 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_322")]; - tensor var_323_dtype_0 = const()[name = tensor("op_323_dtype_0"), val = tensor("fp32")]; - tensor var_324 = const()[name = tensor("op_324"), val = tensor([1, 16, 1])]; - tensor var_323 = cast(dtype = var_323_dtype_0, x = var_322)[name = tensor("cast_157")]; - tensor fill_mask_3 = reshape(shape = var_324, x = var_323)[name = tensor("fill_mask_3")]; - tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_314)[name = tensor("x_expanded_3")]; - tensor var_328 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_328")]; - tensor var_329 = mul(x = input_21, y = var_328)[name = tensor("op_329")]; - tensor var_330 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_330")]; - tensor filled_window_3 = add(x = var_329, y = var_330)[name = tensor("filled_window_3")]; - tensor var_332 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_332")]; - tensor var_333_dtype_0 = const()[name = tensor("op_333_dtype_0"), val = tensor("fp32")]; - tensor var_334 = const()[name = tensor("op_334"), val = tensor([1, 1, 1])]; - tensor var_333 = cast(dtype = var_333_dtype_0, x = var_332)[name = tensor("cast_156")]; - tensor use_roll_3 = reshape(shape = var_334, x = var_333)[name = tensor("use_roll_3")]; - tensor var_336 = sub(x = var_17, y = use_roll_3)[name = tensor("op_336")]; - tensor var_337 = mul(x = filled_window_3, y = var_336)[name = tensor("op_337")]; - tensor var_338 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_338")]; - tensor input_31 = add(x = var_337, y = var_338)[name = tensor("input_31")]; - tensor var_340 = const()[name = tensor("op_340"), val = tensor(0x1p+0)]; - tensor var_341 = add(x = clip_1, y = var_340)[name = tensor("op_341")]; - tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_341)[name = tensor("clip_3")]; + tensor var_373_begin_0 = const()[name = tensor("op_373_begin_0"), val = tensor([0, -1, 0])]; + tensor var_373_end_0 = const()[name = tensor("op_373_end_0"), val = tensor([3, 16, 256])]; + tensor var_373_end_mask_0 = const()[name = tensor("op_373_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_373 = slice_by_index(begin = var_373_begin_0, end = var_373_end_0, end_mask = var_373_end_mask_0, x = conv_out_1)[name = tensor("op_373")]; + tensor var_375_perm_0 = const()[name = tensor("op_375_perm_0"), val = tensor([1, 0, 2])]; + tensor var_375 = transpose(perm = var_375_perm_0, x = var_373)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_375)[name = tensor("input_31")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; + tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; + tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; + tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; + tensor var_398 = const()[name = tensor("op_398"), val = tensor(0x1p-1)]; + tensor var_399 = mul(x = input_39, y = var_398)[name = tensor("op_399")]; + tensor input_41 = add(x = var_399, y = input_31)[name = tensor("input_41")]; + tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; + tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; + tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; + tensor input_45 = layer_norm(axes = input_45_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_43)[name = tensor("input_45")]; + tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; + tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; + tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; + tensor var_428 = const()[name = tensor("op_428"), val = tensor(0x1p-1)]; + tensor var_429 = mul(x = input_51, y = var_428)[name = tensor("op_429")]; + tensor input_53 = add(x = var_429, y = input_43)[name = tensor("input_53")]; tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; - tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_31)[name = tensor("x_7")]; - tensor input_33_perm_0 = const()[name = tensor("input_33_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_9_pad_type_0 = const()[name = tensor("inputs_9_pad_type_0"), val = tensor("valid")]; - tensor inputs_9_strides_0 = const()[name = tensor("inputs_9_strides_0"), val = tensor([1])]; - tensor inputs_9_pad_0 = const()[name = tensor("inputs_9_pad_0"), val = tensor([0, 0])]; - tensor inputs_9_dilations_0 = const()[name = tensor("inputs_9_dilations_0"), val = tensor([1])]; - tensor inputs_9_groups_0 = const()[name = tensor("inputs_9_groups_0"), val = tensor(1)]; - tensor input_33 = transpose(perm = input_33_perm_0, x = x_7)[name = tensor("transpose_64")]; - tensor inputs_9 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_9_dilations_0, groups = inputs_9_groups_0, pad = inputs_9_pad_0, pad_type = inputs_9_pad_type_0, strides = inputs_9_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_33)[name = tensor("inputs_9")]; - tensor var_362_split_sizes_0 = const()[name = tensor("op_362_split_sizes_0"), val = tensor([256, 256])]; - tensor var_362_axis_0 = const()[name = tensor("op_362_axis_0"), val = tensor(1)]; - tensor var_362_0, tensor var_362_1 = split(axis = var_362_axis_0, split_sizes = var_362_split_sizes_0, x = inputs_9)[name = tensor("op_362")]; - tensor var_364 = sigmoid(x = var_362_1)[name = tensor("op_364")]; - tensor inputs_11 = mul(x = var_362_0, y = var_364)[name = tensor("inputs_11")]; - tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; - tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; - tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; - tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_11)[name = tensor("outputs_aug_3")]; - tensor input_35_begin_0 = const()[name = tensor("input_35_begin_0"), val = tensor([0, 0, 0])]; - tensor input_35_end_0 = const()[name = tensor("input_35_end_0"), val = tensor([1, 256, 16])]; - tensor input_35_end_mask_0 = const()[name = tensor("input_35_end_mask_0"), val = tensor([true, true, false])]; - tensor input_35 = slice_by_index(begin = input_35_begin_0, end = input_35_end_0, end_mask = input_35_end_mask_0, x = outputs_aug_3)[name = tensor("input_35")]; - tensor inputs_13 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_35)[name = tensor("inputs_13")]; - tensor input_37 = silu(x = inputs_13)[name = tensor("input_37")]; - tensor input_39_pad_type_0 = const()[name = tensor("input_39_pad_type_0"), val = tensor("valid")]; - tensor input_39_strides_0 = const()[name = tensor("input_39_strides_0"), val = tensor([1])]; - tensor input_39_pad_0 = const()[name = tensor("input_39_pad_0"), val = tensor([0, 0])]; - tensor input_39_dilations_0 = const()[name = tensor("input_39_dilations_0"), val = tensor([1])]; - tensor input_39_groups_0 = const()[name = tensor("input_39_groups_0"), val = tensor(1)]; - tensor input_39 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_37)[name = tensor("input_39")]; - tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_395_begin_0 = const()[name = tensor("op_395_begin_0"), val = tensor([0, -1, 0])]; - tensor var_395_end_0 = const()[name = tensor("op_395_end_0"), val = tensor([1, 16, 256])]; - tensor var_395_end_mask_0 = const()[name = tensor("op_395_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_39)[name = tensor("transpose_63")]; - tensor var_395 = slice_by_index(begin = var_395_begin_0, end = var_395_end_0, end_mask = var_395_end_mask_0, x = conv_out_3)[name = tensor("op_395")]; - tensor var_398_begin_0 = const()[name = tensor("op_398_begin_0"), val = tensor([0, 2, 0])]; - tensor var_398_end_0 = const()[name = tensor("op_398_end_0"), val = tensor([1, 1, 256])]; - tensor var_398_end_mask_0 = const()[name = tensor("op_398_end_mask_0"), val = tensor([true, true, true])]; - tensor var_398 = slice_by_index(begin = var_398_begin_0, end = var_398_end_0, end_mask = var_398_end_mask_0, x = x_3)[name = tensor("op_398")]; - tensor var_401_begin_0 = const()[name = tensor("op_401_begin_0"), val = tensor([0, 1, 0])]; - tensor var_401_end_0 = const()[name = tensor("op_401_end_0"), val = tensor([1, 16, 256])]; - tensor var_401_end_mask_0 = const()[name = tensor("op_401_end_mask_0"), val = tensor([true, true, true])]; - tensor var_401 = slice_by_index(begin = var_401_begin_0, end = var_401_end_0, end_mask = var_401_end_mask_0, x = input_31)[name = tensor("op_401")]; - tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_401, var_398))[name = tensor("rolled_5")]; - tensor clip_4 = clip(alpha = var_21, beta = var_20, x = clip_3)[name = tensor("clip_4")]; - tensor var_406 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_406")]; - tensor var_407_dtype_0 = const()[name = tensor("op_407_dtype_0"), val = tensor("fp32")]; - tensor var_408 = const()[name = tensor("op_408"), val = tensor([1, 16, 1])]; - tensor var_407 = cast(dtype = var_407_dtype_0, x = var_406)[name = tensor("cast_155")]; - tensor fill_mask_5 = reshape(shape = var_408, x = var_407)[name = tensor("fill_mask_5")]; - tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_398)[name = tensor("x_expanded_5")]; - tensor var_412 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_412")]; - tensor var_413 = mul(x = input_31, y = var_412)[name = tensor("op_413")]; - tensor var_414 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_414")]; - tensor filled_window_5 = add(x = var_413, y = var_414)[name = tensor("filled_window_5")]; - tensor var_416 = greater_equal(x = clip_3, y = var_16)[name = tensor("op_416")]; - tensor var_417_dtype_0 = const()[name = tensor("op_417_dtype_0"), val = tensor("fp32")]; - tensor var_418 = const()[name = tensor("op_418"), val = tensor([1, 1, 1])]; - tensor var_417 = cast(dtype = var_417_dtype_0, x = var_416)[name = tensor("cast_154")]; - tensor use_roll_5 = reshape(shape = var_418, x = var_417)[name = tensor("use_roll_5")]; - tensor var_420 = sub(x = var_17, y = use_roll_5)[name = tensor("op_420")]; - tensor var_421 = mul(x = filled_window_5, y = var_420)[name = tensor("op_421")]; - tensor var_422 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_422")]; - tensor input_41 = add(x = var_421, y = var_422)[name = tensor("input_41")]; - tensor var_424 = const()[name = tensor("op_424"), val = tensor(0x1p+0)]; - tensor var_425 = add(x = clip_3, y = var_424)[name = tensor("op_425")]; - tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_425)[name = tensor("clip_5")]; - tensor x_9_axes_0 = const()[name = tensor("x_9_axes_0"), val = tensor([-1])]; - tensor x_9 = layer_norm(axes = x_9_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_41)[name = tensor("x_9")]; - tensor input_43_perm_0 = const()[name = tensor("input_43_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_15_pad_type_0 = const()[name = tensor("inputs_15_pad_type_0"), val = tensor("valid")]; - tensor inputs_15_strides_0 = const()[name = tensor("inputs_15_strides_0"), val = tensor([1])]; - tensor inputs_15_pad_0 = const()[name = tensor("inputs_15_pad_0"), val = tensor([0, 0])]; - tensor inputs_15_dilations_0 = const()[name = tensor("inputs_15_dilations_0"), val = tensor([1])]; - tensor inputs_15_groups_0 = const()[name = tensor("inputs_15_groups_0"), val = tensor(1)]; - tensor input_43 = transpose(perm = input_43_perm_0, x = x_9)[name = tensor("transpose_62")]; - tensor inputs_15 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_15_dilations_0, groups = inputs_15_groups_0, pad = inputs_15_pad_0, pad_type = inputs_15_pad_type_0, strides = inputs_15_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_43)[name = tensor("inputs_15")]; - tensor var_446_split_sizes_0 = const()[name = tensor("op_446_split_sizes_0"), val = tensor([256, 256])]; - tensor var_446_axis_0 = const()[name = tensor("op_446_axis_0"), val = tensor(1)]; - tensor var_446_0, tensor var_446_1 = split(axis = var_446_axis_0, split_sizes = var_446_split_sizes_0, x = inputs_15)[name = tensor("op_446")]; - tensor var_448 = sigmoid(x = var_446_1)[name = tensor("op_448")]; - tensor inputs_17 = mul(x = var_446_0, y = var_448)[name = tensor("inputs_17")]; - tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; - tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; - tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; - tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_17)[name = tensor("outputs_aug_5")]; - tensor input_45_begin_0 = const()[name = tensor("input_45_begin_0"), val = tensor([0, 0, 0])]; - tensor input_45_end_0 = const()[name = tensor("input_45_end_0"), val = tensor([1, 256, 16])]; - tensor input_45_end_mask_0 = const()[name = tensor("input_45_end_mask_0"), val = tensor([true, true, false])]; - tensor input_45 = slice_by_index(begin = input_45_begin_0, end = input_45_end_0, end_mask = input_45_end_mask_0, x = outputs_aug_5)[name = tensor("input_45")]; - tensor inputs_19 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_45)[name = tensor("inputs_19")]; - tensor input_47 = silu(x = inputs_19)[name = tensor("input_47")]; - tensor input_49_pad_type_0 = const()[name = tensor("input_49_pad_type_0"), val = tensor("valid")]; - tensor input_49_strides_0 = const()[name = tensor("input_49_strides_0"), val = tensor([1])]; - tensor input_49_pad_0 = const()[name = tensor("input_49_pad_0"), val = tensor([0, 0])]; - tensor input_49_dilations_0 = const()[name = tensor("input_49_dilations_0"), val = tensor([1])]; - tensor input_49_groups_0 = const()[name = tensor("input_49_groups_0"), val = tensor(1)]; - tensor input_49 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_49_dilations_0, groups = input_49_groups_0, pad = input_49_pad_0, pad_type = input_49_pad_type_0, strides = input_49_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_47)[name = tensor("input_49")]; - tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_479_begin_0 = const()[name = tensor("op_479_begin_0"), val = tensor([0, -1, 0])]; - tensor var_479_end_0 = const()[name = tensor("op_479_end_0"), val = tensor([1, 16, 256])]; - tensor var_479_end_mask_0 = const()[name = tensor("op_479_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_49)[name = tensor("transpose_61")]; - tensor var_479 = slice_by_index(begin = var_479_begin_0, end = var_479_end_0, end_mask = var_479_end_mask_0, x = conv_out_5)[name = tensor("op_479")]; - tensor var_482_interleave_0 = const()[name = tensor("op_482_interleave_0"), val = tensor(false)]; - tensor var_482 = concat(axis = var_33, interleave = var_482_interleave_0, values = (var_311, var_395, var_479))[name = tensor("op_482")]; - tensor input_51 = add(x = x_3, y = var_482)[name = tensor("input_51")]; - tensor input_53_axes_0 = const()[name = tensor("input_53_axes_0"), val = tensor([-1])]; - tensor input_53 = layer_norm(axes = input_53_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_51)[name = tensor("input_53")]; - tensor inputs_21 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_53)[name = tensor("linear_8")]; - tensor input_55 = silu(x = inputs_21)[name = tensor("input_55")]; - tensor input_59 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_55)[name = tensor("linear_9")]; - tensor var_505 = const()[name = tensor("op_505"), val = tensor(0x1p-1)]; - tensor var_506 = mul(x = input_59, y = var_505)[name = tensor("op_506")]; - tensor input_61 = add(x = var_506, y = input_51)[name = tensor("input_61")]; - tensor input_63_axes_0 = const()[name = tensor("input_63_axes_0"), val = tensor([-1])]; - tensor input_63 = layer_norm(axes = input_63_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_61)[name = tensor("input_63")]; - tensor input_65_axes_0 = const()[name = tensor("input_65_axes_0"), val = tensor([-1])]; - tensor input_65 = layer_norm(axes = input_65_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_63)[name = tensor("input_65")]; - tensor inputs_23 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_65)[name = tensor("linear_10")]; - tensor input_67 = silu(x = inputs_23)[name = tensor("input_67")]; - tensor input_71 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_67)[name = tensor("linear_11")]; - tensor var_535 = const()[name = tensor("op_535"), val = tensor(0x1p-1)]; - tensor var_536 = mul(x = input_71, y = var_535)[name = tensor("op_536")]; - tensor input_73 = add(x = var_536, y = input_63)[name = tensor("input_73")]; - tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; - tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_73)[name = tensor("x_11")]; + tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_3_end_0 = const()[name = tensor("prev_kv_3_end_0"), val = tensor([2, 1, 4, 64, 64])]; tensor prev_kv_3_end_mask_0 = const()[name = tensor("prev_kv_3_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -563,316 +488,240 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_550 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_11)[name = tensor("linear_12")]; - tensor var_551 = const()[name = tensor("op_551"), val = tensor([1, 3, 4, 64])]; - tensor var_552 = reshape(shape = var_551, x = var_550)[name = tensor("op_552")]; + tensor var_443 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_444 = const()[name = tensor("op_444"), val = tensor([1, 3, 4, 64])]; + tensor var_445 = reshape(shape = var_444, x = var_443)[name = tensor("op_445")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_556 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_11)[name = tensor("linear_13")]; - tensor var_557 = const()[name = tensor("op_557"), val = tensor(0x1p-3)]; - tensor var_558 = mul(x = var_556, y = var_557)[name = tensor("op_558")]; - tensor var_559 = const()[name = tensor("op_559"), val = tensor([1, 3, 4, 64])]; - tensor var_560 = reshape(shape = var_559, x = var_558)[name = tensor("op_560")]; + tensor var_449 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_450 = const()[name = tensor("op_450"), val = tensor(0x1p-3)]; + tensor var_451 = mul(x = var_449, y = var_450)[name = tensor("op_451")]; + tensor var_452 = const()[name = tensor("op_452"), val = tensor([1, 3, 4, 64])]; + tensor var_453 = reshape(shape = var_452, x = var_451)[name = tensor("op_453")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_564 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_11)[name = tensor("linear_14")]; - tensor var_565 = const()[name = tensor("op_565"), val = tensor([1, 3, 4, 64])]; - tensor var_566 = reshape(shape = var_565, x = var_564)[name = tensor("op_566")]; + tensor var_457 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_458 = const()[name = tensor("op_458"), val = tensor([1, 3, 4, 64])]; + tensor var_459 = reshape(shape = var_458, x = var_457)[name = tensor("op_459")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_77 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_11)[name = tensor("linear_15")]; + tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; tensor s_t_3 = add(x = prev_scale_3, y = encoder__t_index)[name = tensor("s_t_3")]; tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_560)[name = tensor("transpose_59")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_552)[name = tensor("transpose_60")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_453)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_445)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_576 = const()[name = tensor("op_576"), val = tensor([3, 1])]; - tensor var_577 = reshape(shape = var_576, x = sqrt_s_t_3)[name = tensor("op_577")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_577)[name = tensor("M_3")]; - tensor var_579 = mul(x = qk_3, y = M_3)[name = tensor("op_579")]; + tensor var_469 = const()[name = tensor("op_469"), val = tensor([3, 1])]; + tensor var_470 = reshape(shape = var_469, x = sqrt_s_t_3)[name = tensor("op_470")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_470)[name = tensor("M_3")]; + tensor var_472 = mul(x = qk_3, y = M_3)[name = tensor("op_472")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_566)[name = tensor("transpose_58")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_579, y = v_3)[name = tensor("inner_3")]; - tensor var_581_transpose_x_0 = const()[name = tensor("op_581_transpose_x_0"), val = tensor(false)]; - tensor var_581_transpose_y_0 = const()[name = tensor("op_581_transpose_y_0"), val = tensor(false)]; - tensor var_581 = matmul(transpose_x = var_581_transpose_x_0, transpose_y = var_581_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_581")]; - tensor var_582 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_582")]; - tensor var_583 = const()[name = tensor("op_583"), val = tensor([1, 1, 3, 1])]; - tensor var_584 = reshape(shape = var_583, x = var_582)[name = tensor("op_584")]; - tensor cross_3 = mul(x = var_581, y = var_584)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_459)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_472, y = v_3)[name = tensor("inner_3")]; + tensor var_474_transpose_x_0 = const()[name = tensor("op_474_transpose_x_0"), val = tensor(false)]; + tensor var_474_transpose_y_0 = const()[name = tensor("op_474_transpose_y_0"), val = tensor(false)]; + tensor var_474 = matmul(transpose_x = var_474_transpose_x_0, transpose_y = var_474_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_474")]; + tensor var_475 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_475")]; + tensor var_476 = const()[name = tensor("op_476"), val = tensor([1, 1, 3, 1])]; + tensor var_477 = reshape(shape = var_476, x = var_475)[name = tensor("op_477")]; + tensor cross_3 = mul(x = var_474, y = var_477)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_587 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_587")]; - tensor var_589_transpose_x_1 = const()[name = tensor("op_589_transpose_x_1"), val = tensor(true)]; - tensor var_589_transpose_y_1 = const()[name = tensor("op_589_transpose_y_1"), val = tensor(false)]; - tensor var_589 = matmul(transpose_x = var_589_transpose_x_1, transpose_y = var_589_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_589")]; - tensor new_kv_unnorm_3 = add(x = var_587, y = var_589)[name = tensor("new_kv_unnorm_3")]; - tensor var_591 = const()[name = tensor("op_591"), val = tensor(0x1.8p+1)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_591)[name = tensor("new_scale_3")]; - tensor var_593 = sqrt(x = new_scale_3)[name = tensor("op_593")]; - tensor var_594 = real_div(x = new_kv_unnorm_3, y = var_593)[name = tensor("op_594")]; - tensor var_595_perm_0 = const()[name = tensor("op_595_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_480 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_480")]; + tensor var_482_transpose_x_1 = const()[name = tensor("op_482_transpose_x_1"), val = tensor(true)]; + tensor var_482_transpose_y_1 = const()[name = tensor("op_482_transpose_y_1"), val = tensor(false)]; + tensor var_482 = matmul(transpose_x = var_482_transpose_x_1, transpose_y = var_482_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_482")]; + tensor new_kv_unnorm_3 = add(x = var_480, y = var_482)[name = tensor("new_kv_unnorm_3")]; + tensor var_484 = const()[name = tensor("op_484"), val = tensor(0x1.8p+1)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_484)[name = tensor("new_scale_3")]; + tensor var_486 = sqrt(x = new_scale_3)[name = tensor("op_486")]; + tensor var_487 = real_div(x = new_kv_unnorm_3, y = var_486)[name = tensor("op_487")]; + tensor var_488_perm_0 = const()[name = tensor("op_488_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_595 = transpose(perm = var_595_perm_0, x = out_7)[name = tensor("transpose_57")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_595)[name = tensor("out_9")]; - tensor var_599 = const()[name = tensor("op_599"), val = tensor([1, 3, 256])]; - tensor out_11 = reshape(shape = var_599, x = out_9)[name = tensor("out_11")]; - tensor var_601 = silu(x = input_77)[name = tensor("op_601")]; - tensor input_79 = mul(x = var_601, y = out_11)[name = tensor("input_79")]; - tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_79)[name = tensor("linear_16")]; - tensor x_13 = add(x = input_73, y = ret_out_3)[name = tensor("x_13")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor var_488 = transpose(perm = var_488_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_488)[name = tensor("out_9")]; + tensor var_492 = const()[name = tensor("op_492"), val = tensor([1, 3, 256])]; + tensor out_11 = reshape(shape = var_492, x = out_9)[name = tensor("out_11")]; + tensor var_494 = silu(x = input_57)[name = tensor("op_494")]; + tensor input_59 = mul(x = var_494, y = out_11)[name = tensor("input_59")]; + tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; + tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; + tensor window_9_begin_0 = const()[name = tensor("window_9_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_9_end_0 = const()[name = tensor("window_9_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_9_end_mask_0 = const()[name = tensor("window_9_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_9_squeeze_mask_0 = const()[name = tensor("window_9_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_9 = slice_by_index(begin = window_9_begin_0, end = window_9_end_0, end_mask = window_9_end_mask_0, squeeze_mask = window_9_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_9")]; tensor filled_7_begin_0 = const()[name = tensor("filled_7_begin_0"), val = tensor([1, 0])]; tensor filled_7_end_0 = const()[name = tensor("filled_7_end_0"), val = tensor([2, 1])]; tensor filled_7_end_mask_0 = const()[name = tensor("filled_7_end_mask_0"), val = tensor([false, true])]; tensor filled_7_squeeze_mask_0 = const()[name = tensor("filled_7_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_7 = slice_by_index(begin = filled_7_begin_0, end = filled_7_end_0, end_mask = filled_7_end_mask_0, squeeze_mask = filled_7_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_7")]; - tensor var_610_begin_0 = const()[name = tensor("op_610_begin_0"), val = tensor([0, 0, 0])]; - tensor var_610_end_0 = const()[name = tensor("op_610_end_0"), val = tensor([1, 1, 256])]; - tensor var_610_end_mask_0 = const()[name = tensor("op_610_end_mask_0"), val = tensor([true, false, true])]; - tensor var_610 = slice_by_index(begin = var_610_begin_0, end = var_610_end_0, end_mask = var_610_end_mask_0, x = x_13)[name = tensor("op_610")]; - tensor var_613_begin_0 = const()[name = tensor("op_613_begin_0"), val = tensor([0, 1, 0])]; - tensor var_613_end_0 = const()[name = tensor("op_613_end_0"), val = tensor([1, 16, 256])]; - tensor var_613_end_mask_0 = const()[name = tensor("op_613_end_mask_0"), val = tensor([true, true, true])]; - tensor var_613 = slice_by_index(begin = var_613_begin_0, end = var_613_end_0, end_mask = var_613_end_mask_0, x = window_3)[name = tensor("op_613")]; + tensor var_503_begin_0 = const()[name = tensor("op_503_begin_0"), val = tensor([0, 0, 0])]; + tensor var_503_end_0 = const()[name = tensor("op_503_end_0"), val = tensor([1, 1, 256])]; + tensor var_503_end_mask_0 = const()[name = tensor("op_503_end_mask_0"), val = tensor([true, false, true])]; + tensor var_503 = slice_by_index(begin = var_503_begin_0, end = var_503_end_0, end_mask = var_503_end_mask_0, x = x_9)[name = tensor("op_503")]; + tensor var_506_begin_0 = const()[name = tensor("op_506_begin_0"), val = tensor([0, 1, 0])]; + tensor var_506_end_0 = const()[name = tensor("op_506_end_0"), val = tensor([1, 16, 256])]; + tensor var_506_end_mask_0 = const()[name = tensor("op_506_end_mask_0"), val = tensor([true, true, true])]; + tensor var_506 = slice_by_index(begin = var_506_begin_0, end = var_506_end_0, end_mask = var_506_end_mask_0, x = window_9)[name = tensor("op_506")]; tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; - tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_613, var_610))[name = tensor("rolled_7")]; + tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_506, var_503))[name = tensor("rolled_7")]; tensor clip_6 = clip(alpha = var_21, beta = var_20, x = filled_7)[name = tensor("clip_6")]; - tensor var_618 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_618")]; - tensor var_619_dtype_0 = const()[name = tensor("op_619_dtype_0"), val = tensor("fp32")]; - tensor var_620 = const()[name = tensor("op_620"), val = tensor([1, 16, 1])]; - tensor var_619 = cast(dtype = var_619_dtype_0, x = var_618)[name = tensor("cast_153")]; - tensor fill_mask_7 = reshape(shape = var_620, x = var_619)[name = tensor("fill_mask_7")]; + tensor var_511 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_511")]; + tensor var_512_dtype_0 = const()[name = tensor("op_512_dtype_0"), val = tensor("fp32")]; + tensor var_513 = const()[name = tensor("op_513"), val = tensor([1, 16, 1])]; + tensor var_512 = cast(dtype = var_512_dtype_0, x = var_511)[name = tensor("cast_153")]; + tensor fill_mask_7 = reshape(shape = var_513, x = var_512)[name = tensor("fill_mask_7")]; tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_610)[name = tensor("x_expanded_7")]; - tensor var_624 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_624")]; - tensor var_625 = mul(x = window_3, y = var_624)[name = tensor("op_625")]; - tensor var_626 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_626")]; - tensor filled_window_7 = add(x = var_625, y = var_626)[name = tensor("filled_window_7")]; - tensor var_628 = greater_equal(x = filled_7, y = var_16)[name = tensor("op_628")]; - tensor var_629_dtype_0 = const()[name = tensor("op_629_dtype_0"), val = tensor("fp32")]; - tensor var_630 = const()[name = tensor("op_630"), val = tensor([1, 1, 1])]; - tensor var_629 = cast(dtype = var_629_dtype_0, x = var_628)[name = tensor("cast_152")]; - tensor use_roll_7 = reshape(shape = var_630, x = var_629)[name = tensor("use_roll_7")]; - tensor var_632 = sub(x = var_17, y = use_roll_7)[name = tensor("op_632")]; - tensor var_633 = mul(x = filled_window_7, y = var_632)[name = tensor("op_633")]; - tensor var_634 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_634")]; - tensor input_81 = add(x = var_633, y = var_634)[name = tensor("input_81")]; - tensor var_636 = const()[name = tensor("op_636"), val = tensor(0x1p+0)]; - tensor var_637 = add(x = filled_7, y = var_636)[name = tensor("op_637")]; - tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_637)[name = tensor("clip_7")]; - tensor x_15_axes_0 = const()[name = tensor("x_15_axes_0"), val = tensor([-1])]; - tensor x_15 = layer_norm(axes = x_15_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_81)[name = tensor("x_15")]; - tensor input_83_perm_0 = const()[name = tensor("input_83_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_25_pad_type_0 = const()[name = tensor("inputs_25_pad_type_0"), val = tensor("valid")]; - tensor inputs_25_strides_0 = const()[name = tensor("inputs_25_strides_0"), val = tensor([1])]; - tensor inputs_25_pad_0 = const()[name = tensor("inputs_25_pad_0"), val = tensor([0, 0])]; - tensor inputs_25_dilations_0 = const()[name = tensor("inputs_25_dilations_0"), val = tensor([1])]; - tensor inputs_25_groups_0 = const()[name = tensor("inputs_25_groups_0"), val = tensor(1)]; - tensor input_83 = transpose(perm = input_83_perm_0, x = x_15)[name = tensor("transpose_56")]; - tensor inputs_25 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_25_dilations_0, groups = inputs_25_groups_0, pad = inputs_25_pad_0, pad_type = inputs_25_pad_type_0, strides = inputs_25_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_83)[name = tensor("inputs_25")]; - tensor var_658_split_sizes_0 = const()[name = tensor("op_658_split_sizes_0"), val = tensor([256, 256])]; - tensor var_658_axis_0 = const()[name = tensor("op_658_axis_0"), val = tensor(1)]; - tensor var_658_0, tensor var_658_1 = split(axis = var_658_axis_0, split_sizes = var_658_split_sizes_0, x = inputs_25)[name = tensor("op_658")]; - tensor var_660 = sigmoid(x = var_658_1)[name = tensor("op_660")]; - tensor inputs_27 = mul(x = var_658_0, y = var_660)[name = tensor("inputs_27")]; - tensor outputs_aug_7_pad_type_0 = const()[name = tensor("outputs_aug_7_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_7_pad_0 = const()[name = tensor("outputs_aug_7_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_7_groups_0 = const()[name = tensor("outputs_aug_7_groups_0"), val = tensor(256)]; - tensor outputs_aug_7_strides_0 = const()[name = tensor("outputs_aug_7_strides_0"), val = tensor([1])]; - tensor outputs_aug_7_dilations_0 = const()[name = tensor("outputs_aug_7_dilations_0"), val = tensor([1])]; - tensor outputs_aug_7 = conv(dilations = outputs_aug_7_dilations_0, groups = outputs_aug_7_groups_0, pad = outputs_aug_7_pad_0, pad_type = outputs_aug_7_pad_type_0, strides = outputs_aug_7_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_27)[name = tensor("outputs_aug_7")]; - tensor input_85_begin_0 = const()[name = tensor("input_85_begin_0"), val = tensor([0, 0, 0])]; - tensor input_85_end_0 = const()[name = tensor("input_85_end_0"), val = tensor([1, 256, 16])]; - tensor input_85_end_mask_0 = const()[name = tensor("input_85_end_mask_0"), val = tensor([true, true, false])]; - tensor input_85 = slice_by_index(begin = input_85_begin_0, end = input_85_end_0, end_mask = input_85_end_mask_0, x = outputs_aug_7)[name = tensor("input_85")]; - tensor inputs_29 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_85)[name = tensor("inputs_29")]; - tensor input_87 = silu(x = inputs_29)[name = tensor("input_87")]; - tensor input_89_pad_type_0 = const()[name = tensor("input_89_pad_type_0"), val = tensor("valid")]; - tensor input_89_strides_0 = const()[name = tensor("input_89_strides_0"), val = tensor([1])]; - tensor input_89_pad_0 = const()[name = tensor("input_89_pad_0"), val = tensor([0, 0])]; - tensor input_89_dilations_0 = const()[name = tensor("input_89_dilations_0"), val = tensor([1])]; - tensor input_89_groups_0 = const()[name = tensor("input_89_groups_0"), val = tensor(1)]; - tensor input_89 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_89_dilations_0, groups = input_89_groups_0, pad = input_89_pad_0, pad_type = input_89_pad_type_0, strides = input_89_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_87)[name = tensor("input_89")]; - tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_691_begin_0 = const()[name = tensor("op_691_begin_0"), val = tensor([0, -1, 0])]; - tensor var_691_end_0 = const()[name = tensor("op_691_end_0"), val = tensor([1, 16, 256])]; - tensor var_691_end_mask_0 = const()[name = tensor("op_691_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_89)[name = tensor("transpose_55")]; - tensor var_691 = slice_by_index(begin = var_691_begin_0, end = var_691_end_0, end_mask = var_691_end_mask_0, x = conv_out_7)[name = tensor("op_691")]; - tensor var_694_begin_0 = const()[name = tensor("op_694_begin_0"), val = tensor([0, 1, 0])]; - tensor var_694_end_0 = const()[name = tensor("op_694_end_0"), val = tensor([1, 2, 256])]; - tensor var_694_end_mask_0 = const()[name = tensor("op_694_end_mask_0"), val = tensor([true, false, true])]; - tensor var_694 = slice_by_index(begin = var_694_begin_0, end = var_694_end_0, end_mask = var_694_end_mask_0, x = x_13)[name = tensor("op_694")]; - tensor var_697_begin_0 = const()[name = tensor("op_697_begin_0"), val = tensor([0, 1, 0])]; - tensor var_697_end_0 = const()[name = tensor("op_697_end_0"), val = tensor([1, 16, 256])]; - tensor var_697_end_mask_0 = const()[name = tensor("op_697_end_mask_0"), val = tensor([true, true, true])]; - tensor var_697 = slice_by_index(begin = var_697_begin_0, end = var_697_end_0, end_mask = var_697_end_mask_0, x = input_81)[name = tensor("op_697")]; + tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_503)[name = tensor("x_expanded_7")]; + tensor var_517 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_517")]; + tensor var_518 = mul(x = window_9, y = var_517)[name = tensor("op_518")]; + tensor var_519 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_519")]; + tensor filled_window_7 = add(x = var_518, y = var_519)[name = tensor("filled_window_7")]; + tensor var_521 = greater_equal(x = filled_7, y = var_16)[name = tensor("op_521")]; + tensor var_522_dtype_0 = const()[name = tensor("op_522_dtype_0"), val = tensor("fp32")]; + tensor var_523 = const()[name = tensor("op_523"), val = tensor([1, 1, 1])]; + tensor var_522 = cast(dtype = var_522_dtype_0, x = var_521)[name = tensor("cast_152")]; + tensor use_roll_7 = reshape(shape = var_523, x = var_522)[name = tensor("use_roll_7")]; + tensor var_525 = sub(x = var_17, y = use_roll_7)[name = tensor("op_525")]; + tensor var_526 = mul(x = filled_window_7, y = var_525)[name = tensor("op_526")]; + tensor var_527 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_527")]; + tensor window_11 = add(x = var_526, y = var_527)[name = tensor("window_11")]; + tensor var_529 = const()[name = tensor("op_529"), val = tensor(0x1p+0)]; + tensor var_530 = add(x = filled_7, y = var_529)[name = tensor("op_530")]; + tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_530)[name = tensor("clip_7")]; + tensor var_533_begin_0 = const()[name = tensor("op_533_begin_0"), val = tensor([0, 1, 0])]; + tensor var_533_end_0 = const()[name = tensor("op_533_end_0"), val = tensor([1, 2, 256])]; + tensor var_533_end_mask_0 = const()[name = tensor("op_533_end_mask_0"), val = tensor([true, false, true])]; + tensor var_533 = slice_by_index(begin = var_533_begin_0, end = var_533_end_0, end_mask = var_533_end_mask_0, x = x_9)[name = tensor("op_533")]; + tensor var_536_begin_0 = const()[name = tensor("op_536_begin_0"), val = tensor([0, 1, 0])]; + tensor var_536_end_0 = const()[name = tensor("op_536_end_0"), val = tensor([1, 16, 256])]; + tensor var_536_end_mask_0 = const()[name = tensor("op_536_end_mask_0"), val = tensor([true, true, true])]; + tensor var_536 = slice_by_index(begin = var_536_begin_0, end = var_536_end_0, end_mask = var_536_end_mask_0, x = window_11)[name = tensor("op_536")]; tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; - tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_697, var_694))[name = tensor("rolled_9")]; + tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_536, var_533))[name = tensor("rolled_9")]; tensor clip_8 = clip(alpha = var_21, beta = var_20, x = clip_7)[name = tensor("clip_8")]; - tensor var_702 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_702")]; - tensor var_703_dtype_0 = const()[name = tensor("op_703_dtype_0"), val = tensor("fp32")]; - tensor var_704 = const()[name = tensor("op_704"), val = tensor([1, 16, 1])]; - tensor var_703 = cast(dtype = var_703_dtype_0, x = var_702)[name = tensor("cast_151")]; - tensor fill_mask_9 = reshape(shape = var_704, x = var_703)[name = tensor("fill_mask_9")]; + tensor var_541 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_541")]; + tensor var_542_dtype_0 = const()[name = tensor("op_542_dtype_0"), val = tensor("fp32")]; + tensor var_543 = const()[name = tensor("op_543"), val = tensor([1, 16, 1])]; + tensor var_542 = cast(dtype = var_542_dtype_0, x = var_541)[name = tensor("cast_151")]; + tensor fill_mask_9 = reshape(shape = var_543, x = var_542)[name = tensor("fill_mask_9")]; tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_694)[name = tensor("x_expanded_9")]; - tensor var_708 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_708")]; - tensor var_709 = mul(x = input_81, y = var_708)[name = tensor("op_709")]; - tensor var_710 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_710")]; - tensor filled_window_9 = add(x = var_709, y = var_710)[name = tensor("filled_window_9")]; - tensor var_712 = greater_equal(x = clip_7, y = var_16)[name = tensor("op_712")]; - tensor var_713_dtype_0 = const()[name = tensor("op_713_dtype_0"), val = tensor("fp32")]; - tensor var_714 = const()[name = tensor("op_714"), val = tensor([1, 1, 1])]; - tensor var_713 = cast(dtype = var_713_dtype_0, x = var_712)[name = tensor("cast_150")]; - tensor use_roll_9 = reshape(shape = var_714, x = var_713)[name = tensor("use_roll_9")]; - tensor var_716 = sub(x = var_17, y = use_roll_9)[name = tensor("op_716")]; - tensor var_717 = mul(x = filled_window_9, y = var_716)[name = tensor("op_717")]; - tensor var_718 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_718")]; - tensor input_91 = add(x = var_717, y = var_718)[name = tensor("input_91")]; - tensor var_720 = const()[name = tensor("op_720"), val = tensor(0x1p+0)]; - tensor var_721 = add(x = clip_7, y = var_720)[name = tensor("op_721")]; - tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_721)[name = tensor("clip_9")]; - tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; - tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_91)[name = tensor("x_17")]; - tensor input_93_perm_0 = const()[name = tensor("input_93_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_31_pad_type_0 = const()[name = tensor("inputs_31_pad_type_0"), val = tensor("valid")]; - tensor inputs_31_strides_0 = const()[name = tensor("inputs_31_strides_0"), val = tensor([1])]; - tensor inputs_31_pad_0 = const()[name = tensor("inputs_31_pad_0"), val = tensor([0, 0])]; - tensor inputs_31_dilations_0 = const()[name = tensor("inputs_31_dilations_0"), val = tensor([1])]; - tensor inputs_31_groups_0 = const()[name = tensor("inputs_31_groups_0"), val = tensor(1)]; - tensor input_93 = transpose(perm = input_93_perm_0, x = x_17)[name = tensor("transpose_54")]; - tensor inputs_31 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_31_dilations_0, groups = inputs_31_groups_0, pad = inputs_31_pad_0, pad_type = inputs_31_pad_type_0, strides = inputs_31_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_93)[name = tensor("inputs_31")]; - tensor var_742_split_sizes_0 = const()[name = tensor("op_742_split_sizes_0"), val = tensor([256, 256])]; - tensor var_742_axis_0 = const()[name = tensor("op_742_axis_0"), val = tensor(1)]; - tensor var_742_0, tensor var_742_1 = split(axis = var_742_axis_0, split_sizes = var_742_split_sizes_0, x = inputs_31)[name = tensor("op_742")]; - tensor var_744 = sigmoid(x = var_742_1)[name = tensor("op_744")]; - tensor inputs_33 = mul(x = var_742_0, y = var_744)[name = tensor("inputs_33")]; - tensor outputs_aug_9_pad_type_0 = const()[name = tensor("outputs_aug_9_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_9_pad_0 = const()[name = tensor("outputs_aug_9_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_9_groups_0 = const()[name = tensor("outputs_aug_9_groups_0"), val = tensor(256)]; - tensor outputs_aug_9_strides_0 = const()[name = tensor("outputs_aug_9_strides_0"), val = tensor([1])]; - tensor outputs_aug_9_dilations_0 = const()[name = tensor("outputs_aug_9_dilations_0"), val = tensor([1])]; - tensor outputs_aug_9 = conv(dilations = outputs_aug_9_dilations_0, groups = outputs_aug_9_groups_0, pad = outputs_aug_9_pad_0, pad_type = outputs_aug_9_pad_type_0, strides = outputs_aug_9_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_33)[name = tensor("outputs_aug_9")]; - tensor input_95_begin_0 = const()[name = tensor("input_95_begin_0"), val = tensor([0, 0, 0])]; - tensor input_95_end_0 = const()[name = tensor("input_95_end_0"), val = tensor([1, 256, 16])]; - tensor input_95_end_mask_0 = const()[name = tensor("input_95_end_mask_0"), val = tensor([true, true, false])]; - tensor input_95 = slice_by_index(begin = input_95_begin_0, end = input_95_end_0, end_mask = input_95_end_mask_0, x = outputs_aug_9)[name = tensor("input_95")]; - tensor inputs_35 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_95)[name = tensor("inputs_35")]; - tensor input_97 = silu(x = inputs_35)[name = tensor("input_97")]; - tensor input_99_pad_type_0 = const()[name = tensor("input_99_pad_type_0"), val = tensor("valid")]; - tensor input_99_strides_0 = const()[name = tensor("input_99_strides_0"), val = tensor([1])]; - tensor input_99_pad_0 = const()[name = tensor("input_99_pad_0"), val = tensor([0, 0])]; - tensor input_99_dilations_0 = const()[name = tensor("input_99_dilations_0"), val = tensor([1])]; - tensor input_99_groups_0 = const()[name = tensor("input_99_groups_0"), val = tensor(1)]; - tensor input_99 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_99_dilations_0, groups = input_99_groups_0, pad = input_99_pad_0, pad_type = input_99_pad_type_0, strides = input_99_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_97)[name = tensor("input_99")]; - tensor conv_out_9_perm_0 = const()[name = tensor("conv_out_9_perm_0"), val = tensor([0, 2, 1])]; - tensor var_775_begin_0 = const()[name = tensor("op_775_begin_0"), val = tensor([0, -1, 0])]; - tensor var_775_end_0 = const()[name = tensor("op_775_end_0"), val = tensor([1, 16, 256])]; - tensor var_775_end_mask_0 = const()[name = tensor("op_775_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_9 = transpose(perm = conv_out_9_perm_0, x = input_99)[name = tensor("transpose_53")]; - tensor var_775 = slice_by_index(begin = var_775_begin_0, end = var_775_end_0, end_mask = var_775_end_mask_0, x = conv_out_9)[name = tensor("op_775")]; - tensor var_778_begin_0 = const()[name = tensor("op_778_begin_0"), val = tensor([0, 2, 0])]; - tensor var_778_end_0 = const()[name = tensor("op_778_end_0"), val = tensor([1, 1, 256])]; - tensor var_778_end_mask_0 = const()[name = tensor("op_778_end_mask_0"), val = tensor([true, true, true])]; - tensor var_778 = slice_by_index(begin = var_778_begin_0, end = var_778_end_0, end_mask = var_778_end_mask_0, x = x_13)[name = tensor("op_778")]; - tensor var_781_begin_0 = const()[name = tensor("op_781_begin_0"), val = tensor([0, 1, 0])]; - tensor var_781_end_0 = const()[name = tensor("op_781_end_0"), val = tensor([1, 16, 256])]; - tensor var_781_end_mask_0 = const()[name = tensor("op_781_end_mask_0"), val = tensor([true, true, true])]; - tensor var_781 = slice_by_index(begin = var_781_begin_0, end = var_781_end_0, end_mask = var_781_end_mask_0, x = input_91)[name = tensor("op_781")]; + tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_533)[name = tensor("x_expanded_9")]; + tensor var_547 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_547")]; + tensor var_548 = mul(x = window_11, y = var_547)[name = tensor("op_548")]; + tensor var_549 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_549")]; + tensor filled_window_9 = add(x = var_548, y = var_549)[name = tensor("filled_window_9")]; + tensor var_551 = greater_equal(x = clip_7, y = var_16)[name = tensor("op_551")]; + tensor var_552_dtype_0 = const()[name = tensor("op_552_dtype_0"), val = tensor("fp32")]; + tensor var_553 = const()[name = tensor("op_553"), val = tensor([1, 1, 1])]; + tensor var_552 = cast(dtype = var_552_dtype_0, x = var_551)[name = tensor("cast_150")]; + tensor use_roll_9 = reshape(shape = var_553, x = var_552)[name = tensor("use_roll_9")]; + tensor var_555 = sub(x = var_17, y = use_roll_9)[name = tensor("op_555")]; + tensor var_556 = mul(x = filled_window_9, y = var_555)[name = tensor("op_556")]; + tensor var_557 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_557")]; + tensor window_13 = add(x = var_556, y = var_557)[name = tensor("window_13")]; + tensor var_559 = const()[name = tensor("op_559"), val = tensor(0x1p+0)]; + tensor var_560 = add(x = clip_7, y = var_559)[name = tensor("op_560")]; + tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_560)[name = tensor("clip_9")]; + tensor var_563_begin_0 = const()[name = tensor("op_563_begin_0"), val = tensor([0, 2, 0])]; + tensor var_563_end_0 = const()[name = tensor("op_563_end_0"), val = tensor([1, 1, 256])]; + tensor var_563_end_mask_0 = const()[name = tensor("op_563_end_mask_0"), val = tensor([true, true, true])]; + tensor var_563 = slice_by_index(begin = var_563_begin_0, end = var_563_end_0, end_mask = var_563_end_mask_0, x = x_9)[name = tensor("op_563")]; + tensor var_566_begin_0 = const()[name = tensor("op_566_begin_0"), val = tensor([0, 1, 0])]; + tensor var_566_end_0 = const()[name = tensor("op_566_end_0"), val = tensor([1, 16, 256])]; + tensor var_566_end_mask_0 = const()[name = tensor("op_566_end_mask_0"), val = tensor([true, true, true])]; + tensor var_566 = slice_by_index(begin = var_566_begin_0, end = var_566_end_0, end_mask = var_566_end_mask_0, x = window_13)[name = tensor("op_566")]; tensor rolled_11_interleave_0 = const()[name = tensor("rolled_11_interleave_0"), val = tensor(false)]; - tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_781, var_778))[name = tensor("rolled_11")]; + tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_566, var_563))[name = tensor("rolled_11")]; tensor clip_10 = clip(alpha = var_21, beta = var_20, x = clip_9)[name = tensor("clip_10")]; - tensor var_786 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_786")]; - tensor var_787_dtype_0 = const()[name = tensor("op_787_dtype_0"), val = tensor("fp32")]; - tensor var_788 = const()[name = tensor("op_788"), val = tensor([1, 16, 1])]; - tensor var_787 = cast(dtype = var_787_dtype_0, x = var_786)[name = tensor("cast_149")]; - tensor fill_mask_11 = reshape(shape = var_788, x = var_787)[name = tensor("fill_mask_11")]; + tensor var_571 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_571")]; + tensor var_572_dtype_0 = const()[name = tensor("op_572_dtype_0"), val = tensor("fp32")]; + tensor var_573 = const()[name = tensor("op_573"), val = tensor([1, 16, 1])]; + tensor var_572 = cast(dtype = var_572_dtype_0, x = var_571)[name = tensor("cast_149")]; + tensor fill_mask_11 = reshape(shape = var_573, x = var_572)[name = tensor("fill_mask_11")]; tensor x_expanded_11_reps_0 = const()[name = tensor("x_expanded_11_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_778)[name = tensor("x_expanded_11")]; - tensor var_792 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_792")]; - tensor var_793 = mul(x = input_91, y = var_792)[name = tensor("op_793")]; - tensor var_794 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_794")]; - tensor filled_window_11 = add(x = var_793, y = var_794)[name = tensor("filled_window_11")]; - tensor var_796 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_796")]; - tensor var_797_dtype_0 = const()[name = tensor("op_797_dtype_0"), val = tensor("fp32")]; - tensor var_798 = const()[name = tensor("op_798"), val = tensor([1, 1, 1])]; - tensor var_797 = cast(dtype = var_797_dtype_0, x = var_796)[name = tensor("cast_148")]; - tensor use_roll_11 = reshape(shape = var_798, x = var_797)[name = tensor("use_roll_11")]; - tensor var_800 = sub(x = var_17, y = use_roll_11)[name = tensor("op_800")]; - tensor var_801 = mul(x = filled_window_11, y = var_800)[name = tensor("op_801")]; - tensor var_802 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_802")]; - tensor input_101 = add(x = var_801, y = var_802)[name = tensor("input_101")]; - tensor var_804 = const()[name = tensor("op_804"), val = tensor(0x1p+0)]; - tensor var_805 = add(x = clip_9, y = var_804)[name = tensor("op_805")]; - tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_805)[name = tensor("clip_11")]; - tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; - tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_101)[name = tensor("x_19")]; - tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_37_pad_type_0 = const()[name = tensor("inputs_37_pad_type_0"), val = tensor("valid")]; - tensor inputs_37_strides_0 = const()[name = tensor("inputs_37_strides_0"), val = tensor([1])]; - tensor inputs_37_pad_0 = const()[name = tensor("inputs_37_pad_0"), val = tensor([0, 0])]; - tensor inputs_37_dilations_0 = const()[name = tensor("inputs_37_dilations_0"), val = tensor([1])]; - tensor inputs_37_groups_0 = const()[name = tensor("inputs_37_groups_0"), val = tensor(1)]; - tensor input_103 = transpose(perm = input_103_perm_0, x = x_19)[name = tensor("transpose_52")]; - tensor inputs_37 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_37_dilations_0, groups = inputs_37_groups_0, pad = inputs_37_pad_0, pad_type = inputs_37_pad_type_0, strides = inputs_37_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_37")]; - tensor var_826_split_sizes_0 = const()[name = tensor("op_826_split_sizes_0"), val = tensor([256, 256])]; - tensor var_826_axis_0 = const()[name = tensor("op_826_axis_0"), val = tensor(1)]; - tensor var_826_0, tensor var_826_1 = split(axis = var_826_axis_0, split_sizes = var_826_split_sizes_0, x = inputs_37)[name = tensor("op_826")]; - tensor var_828 = sigmoid(x = var_826_1)[name = tensor("op_828")]; - tensor inputs_39 = mul(x = var_826_0, y = var_828)[name = tensor("inputs_39")]; - tensor outputs_aug_11_pad_type_0 = const()[name = tensor("outputs_aug_11_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_11_pad_0 = const()[name = tensor("outputs_aug_11_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_11_groups_0 = const()[name = tensor("outputs_aug_11_groups_0"), val = tensor(256)]; - tensor outputs_aug_11_strides_0 = const()[name = tensor("outputs_aug_11_strides_0"), val = tensor([1])]; - tensor outputs_aug_11_dilations_0 = const()[name = tensor("outputs_aug_11_dilations_0"), val = tensor([1])]; - tensor outputs_aug_11 = conv(dilations = outputs_aug_11_dilations_0, groups = outputs_aug_11_groups_0, pad = outputs_aug_11_pad_0, pad_type = outputs_aug_11_pad_type_0, strides = outputs_aug_11_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_39)[name = tensor("outputs_aug_11")]; - tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; - tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([1, 256, 16])]; - tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; - tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_11)[name = tensor("input_105")]; - tensor inputs_41 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_105)[name = tensor("inputs_41")]; - tensor input_107 = silu(x = inputs_41)[name = tensor("input_107")]; - tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; - tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; - tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; - tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; - tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; - tensor input_109 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; - tensor conv_out_11_perm_0 = const()[name = tensor("conv_out_11_perm_0"), val = tensor([0, 2, 1])]; - tensor var_859_begin_0 = const()[name = tensor("op_859_begin_0"), val = tensor([0, -1, 0])]; - tensor var_859_end_0 = const()[name = tensor("op_859_end_0"), val = tensor([1, 16, 256])]; - tensor var_859_end_mask_0 = const()[name = tensor("op_859_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_11 = transpose(perm = conv_out_11_perm_0, x = input_109)[name = tensor("transpose_51")]; - tensor var_859 = slice_by_index(begin = var_859_begin_0, end = var_859_end_0, end_mask = var_859_end_mask_0, x = conv_out_11)[name = tensor("op_859")]; - tensor var_862_interleave_0 = const()[name = tensor("op_862_interleave_0"), val = tensor(false)]; - tensor var_862 = concat(axis = var_33, interleave = var_862_interleave_0, values = (var_691, var_775, var_859))[name = tensor("op_862")]; - tensor input_111 = add(x = x_13, y = var_862)[name = tensor("input_111")]; - tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; - tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; - tensor inputs_43 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_17")]; - tensor input_115 = silu(x = inputs_43)[name = tensor("input_115")]; - tensor input_119 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_18")]; - tensor var_885 = const()[name = tensor("op_885"), val = tensor(0x1p-1)]; - tensor var_886 = mul(x = input_119, y = var_885)[name = tensor("op_886")]; - tensor input_121 = add(x = var_886, y = input_111)[name = tensor("input_121")]; - tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; - tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_121)[name = tensor("input_123")]; - tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; - tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; - tensor inputs_45 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_19")]; - tensor input_127 = silu(x = inputs_45)[name = tensor("input_127")]; - tensor input_131 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_20")]; - tensor var_915 = const()[name = tensor("op_915"), val = tensor(0x1p-1)]; - tensor var_916 = mul(x = input_131, y = var_915)[name = tensor("op_916")]; - tensor input_133 = add(x = var_916, y = input_123)[name = tensor("input_133")]; - tensor x_21_axes_0 = const()[name = tensor("x_21_axes_0"), val = tensor([-1])]; - tensor x_21 = layer_norm(axes = x_21_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_133)[name = tensor("x_21")]; + tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_563)[name = tensor("x_expanded_11")]; + tensor var_577 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_577")]; + tensor var_578 = mul(x = window_13, y = var_577)[name = tensor("op_578")]; + tensor var_579 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_579")]; + tensor filled_window_11 = add(x = var_578, y = var_579)[name = tensor("filled_window_11")]; + tensor var_581 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_581")]; + tensor var_582_dtype_0 = const()[name = tensor("op_582_dtype_0"), val = tensor("fp32")]; + tensor var_583 = const()[name = tensor("op_583"), val = tensor([1, 1, 1])]; + tensor var_582 = cast(dtype = var_582_dtype_0, x = var_581)[name = tensor("cast_148")]; + tensor use_roll_11 = reshape(shape = var_583, x = var_582)[name = tensor("use_roll_11")]; + tensor var_585 = sub(x = var_17, y = use_roll_11)[name = tensor("op_585")]; + tensor var_586 = mul(x = filled_window_11, y = var_585)[name = tensor("op_586")]; + tensor var_587 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_587")]; + tensor window_15 = add(x = var_586, y = var_587)[name = tensor("window_15")]; + tensor var_589 = const()[name = tensor("op_589"), val = tensor(0x1p+0)]; + tensor var_590 = add(x = clip_9, y = var_589)[name = tensor("op_590")]; + tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_590)[name = tensor("clip_11")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_30, interleave = input_61_interleave_0, values = (window_11, window_13, window_15))[name = tensor("input_61")]; + tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; + tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; + tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_13_pad_type_0 = const()[name = tensor("inputs_13_pad_type_0"), val = tensor("valid")]; + tensor inputs_13_strides_0 = const()[name = tensor("inputs_13_strides_0"), val = tensor([1])]; + tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; + tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; + tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; + tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; + tensor var_613_split_sizes_0 = const()[name = tensor("op_613_split_sizes_0"), val = tensor([256, 256])]; + tensor var_613_axis_0 = const()[name = tensor("op_613_axis_0"), val = tensor(1)]; + tensor var_613_0, tensor var_613_1 = split(axis = var_613_axis_0, split_sizes = var_613_split_sizes_0, x = inputs_13)[name = tensor("op_613")]; + tensor var_615 = sigmoid(x = var_613_1)[name = tensor("op_615")]; + tensor inputs_15 = mul(x = var_613_0, y = var_615)[name = tensor("inputs_15")]; + tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; + tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; + tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; + tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_15)[name = tensor("outputs_aug_3")]; + tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; + tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([3, 256, 16])]; + tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; + tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_3)[name = tensor("input_65")]; + tensor inputs_17 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_65)[name = tensor("inputs_17")]; + tensor input_67 = silu(x = inputs_17)[name = tensor("input_67")]; + tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; + tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; + tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; + tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; + tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; + tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; + tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; + tensor var_646_begin_0 = const()[name = tensor("op_646_begin_0"), val = tensor([0, -1, 0])]; + tensor var_646_end_0 = const()[name = tensor("op_646_end_0"), val = tensor([3, 16, 256])]; + tensor var_646_end_mask_0 = const()[name = tensor("op_646_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_646 = slice_by_index(begin = var_646_begin_0, end = var_646_end_0, end_mask = var_646_end_mask_0, x = conv_out_3)[name = tensor("op_646")]; + tensor var_648_perm_0 = const()[name = tensor("op_648_perm_0"), val = tensor([1, 0, 2])]; + tensor var_648 = transpose(perm = var_648_perm_0, x = var_646)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_648)[name = tensor("input_71")]; + tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; + tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; + tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; + tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; + tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; + tensor var_671 = const()[name = tensor("op_671"), val = tensor(0x1p-1)]; + tensor var_672 = mul(x = input_79, y = var_671)[name = tensor("op_672")]; + tensor input_81 = add(x = var_672, y = input_71)[name = tensor("input_81")]; + tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; + tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; + tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; + tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; + tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; + tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; + tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; + tensor var_701 = const()[name = tensor("op_701"), val = tensor(0x1p-1)]; + tensor var_702 = mul(x = input_91, y = var_701)[name = tensor("op_702")]; + tensor input_93 = add(x = var_702, y = input_83)[name = tensor("input_93")]; + tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; + tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; tensor prev_kv_5_end_0 = const()[name = tensor("prev_kv_5_end_0"), val = tensor([3, 1, 4, 64, 64])]; tensor prev_kv_5_end_mask_0 = const()[name = tensor("prev_kv_5_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -883,316 +732,240 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_930 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_21)[name = tensor("linear_21")]; - tensor var_931 = const()[name = tensor("op_931"), val = tensor([1, 3, 4, 64])]; - tensor var_932 = reshape(shape = var_931, x = var_930)[name = tensor("op_932")]; + tensor var_716 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_717 = const()[name = tensor("op_717"), val = tensor([1, 3, 4, 64])]; + tensor var_718 = reshape(shape = var_717, x = var_716)[name = tensor("op_718")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_936 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_21)[name = tensor("linear_22")]; - tensor var_937 = const()[name = tensor("op_937"), val = tensor(0x1p-3)]; - tensor var_938 = mul(x = var_936, y = var_937)[name = tensor("op_938")]; - tensor var_939 = const()[name = tensor("op_939"), val = tensor([1, 3, 4, 64])]; - tensor var_940 = reshape(shape = var_939, x = var_938)[name = tensor("op_940")]; + tensor var_722 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_723 = const()[name = tensor("op_723"), val = tensor(0x1p-3)]; + tensor var_724 = mul(x = var_722, y = var_723)[name = tensor("op_724")]; + tensor var_725 = const()[name = tensor("op_725"), val = tensor([1, 3, 4, 64])]; + tensor var_726 = reshape(shape = var_725, x = var_724)[name = tensor("op_726")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_944 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_21)[name = tensor("linear_23")]; - tensor var_945 = const()[name = tensor("op_945"), val = tensor([1, 3, 4, 64])]; - tensor var_946 = reshape(shape = var_945, x = var_944)[name = tensor("op_946")]; + tensor var_730 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_731 = const()[name = tensor("op_731"), val = tensor([1, 3, 4, 64])]; + tensor var_732 = reshape(shape = var_731, x = var_730)[name = tensor("op_732")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_137 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_21)[name = tensor("linear_24")]; + tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; tensor s_t_5 = add(x = prev_scale_5, y = encoder__t_index)[name = tensor("s_t_5")]; tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_940)[name = tensor("transpose_49")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_932)[name = tensor("transpose_50")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_726)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_718)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_956 = const()[name = tensor("op_956"), val = tensor([3, 1])]; - tensor var_957 = reshape(shape = var_956, x = sqrt_s_t_5)[name = tensor("op_957")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_957)[name = tensor("M_5")]; - tensor var_959 = mul(x = qk_5, y = M_5)[name = tensor("op_959")]; + tensor var_742 = const()[name = tensor("op_742"), val = tensor([3, 1])]; + tensor var_743 = reshape(shape = var_742, x = sqrt_s_t_5)[name = tensor("op_743")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_743)[name = tensor("M_5")]; + tensor var_745 = mul(x = qk_5, y = M_5)[name = tensor("op_745")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_946)[name = tensor("transpose_48")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_959, y = v_5)[name = tensor("inner_5")]; - tensor var_961_transpose_x_0 = const()[name = tensor("op_961_transpose_x_0"), val = tensor(false)]; - tensor var_961_transpose_y_0 = const()[name = tensor("op_961_transpose_y_0"), val = tensor(false)]; - tensor var_961 = matmul(transpose_x = var_961_transpose_x_0, transpose_y = var_961_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_961")]; - tensor var_962 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_962")]; - tensor var_963 = const()[name = tensor("op_963"), val = tensor([1, 1, 3, 1])]; - tensor var_964 = reshape(shape = var_963, x = var_962)[name = tensor("op_964")]; - tensor cross_5 = mul(x = var_961, y = var_964)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_732)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_745, y = v_5)[name = tensor("inner_5")]; + tensor var_747_transpose_x_0 = const()[name = tensor("op_747_transpose_x_0"), val = tensor(false)]; + tensor var_747_transpose_y_0 = const()[name = tensor("op_747_transpose_y_0"), val = tensor(false)]; + tensor var_747 = matmul(transpose_x = var_747_transpose_x_0, transpose_y = var_747_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_747")]; + tensor var_748 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_748")]; + tensor var_749 = const()[name = tensor("op_749"), val = tensor([1, 1, 3, 1])]; + tensor var_750 = reshape(shape = var_749, x = var_748)[name = tensor("op_750")]; + tensor cross_5 = mul(x = var_747, y = var_750)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_967 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_967")]; - tensor var_969_transpose_x_1 = const()[name = tensor("op_969_transpose_x_1"), val = tensor(true)]; - tensor var_969_transpose_y_1 = const()[name = tensor("op_969_transpose_y_1"), val = tensor(false)]; - tensor var_969 = matmul(transpose_x = var_969_transpose_x_1, transpose_y = var_969_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_969")]; - tensor new_kv_unnorm_5 = add(x = var_967, y = var_969)[name = tensor("new_kv_unnorm_5")]; - tensor var_971 = const()[name = tensor("op_971"), val = tensor(0x1.8p+1)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_971)[name = tensor("new_scale_5")]; - tensor var_973 = sqrt(x = new_scale_5)[name = tensor("op_973")]; - tensor var_974 = real_div(x = new_kv_unnorm_5, y = var_973)[name = tensor("op_974")]; - tensor var_975_perm_0 = const()[name = tensor("op_975_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_753 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_753")]; + tensor var_755_transpose_x_1 = const()[name = tensor("op_755_transpose_x_1"), val = tensor(true)]; + tensor var_755_transpose_y_1 = const()[name = tensor("op_755_transpose_y_1"), val = tensor(false)]; + tensor var_755 = matmul(transpose_x = var_755_transpose_x_1, transpose_y = var_755_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_755")]; + tensor new_kv_unnorm_5 = add(x = var_753, y = var_755)[name = tensor("new_kv_unnorm_5")]; + tensor var_757 = const()[name = tensor("op_757"), val = tensor(0x1.8p+1)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_757)[name = tensor("new_scale_5")]; + tensor var_759 = sqrt(x = new_scale_5)[name = tensor("op_759")]; + tensor var_760 = real_div(x = new_kv_unnorm_5, y = var_759)[name = tensor("op_760")]; + tensor var_761_perm_0 = const()[name = tensor("op_761_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_975 = transpose(perm = var_975_perm_0, x = out_13)[name = tensor("transpose_47")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_975)[name = tensor("out_15")]; - tensor var_979 = const()[name = tensor("op_979"), val = tensor([1, 3, 256])]; - tensor out_17 = reshape(shape = var_979, x = out_15)[name = tensor("out_17")]; - tensor var_981 = silu(x = input_137)[name = tensor("op_981")]; - tensor input_139 = mul(x = var_981, y = out_17)[name = tensor("input_139")]; - tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_139)[name = tensor("linear_25")]; - tensor x_23 = add(x = input_133, y = ret_out_5)[name = tensor("x_23")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor var_761 = transpose(perm = var_761_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_761)[name = tensor("out_15")]; + tensor var_765 = const()[name = tensor("op_765"), val = tensor([1, 3, 256])]; + tensor out_17 = reshape(shape = var_765, x = out_15)[name = tensor("out_17")]; + tensor var_767 = silu(x = input_97)[name = tensor("op_767")]; + tensor input_99 = mul(x = var_767, y = out_17)[name = tensor("input_99")]; + tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; + tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; + tensor window_17_begin_0 = const()[name = tensor("window_17_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_17_end_0 = const()[name = tensor("window_17_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_17_end_mask_0 = const()[name = tensor("window_17_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_17_squeeze_mask_0 = const()[name = tensor("window_17_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_17 = slice_by_index(begin = window_17_begin_0, end = window_17_end_0, end_mask = window_17_end_mask_0, squeeze_mask = window_17_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_17")]; tensor filled_13_begin_0 = const()[name = tensor("filled_13_begin_0"), val = tensor([2, 0])]; tensor filled_13_end_0 = const()[name = tensor("filled_13_end_0"), val = tensor([3, 1])]; tensor filled_13_end_mask_0 = const()[name = tensor("filled_13_end_mask_0"), val = tensor([false, true])]; tensor filled_13_squeeze_mask_0 = const()[name = tensor("filled_13_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_13 = slice_by_index(begin = filled_13_begin_0, end = filled_13_end_0, end_mask = filled_13_end_mask_0, squeeze_mask = filled_13_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_13")]; - tensor var_990_begin_0 = const()[name = tensor("op_990_begin_0"), val = tensor([0, 0, 0])]; - tensor var_990_end_0 = const()[name = tensor("op_990_end_0"), val = tensor([1, 1, 256])]; - tensor var_990_end_mask_0 = const()[name = tensor("op_990_end_mask_0"), val = tensor([true, false, true])]; - tensor var_990 = slice_by_index(begin = var_990_begin_0, end = var_990_end_0, end_mask = var_990_end_mask_0, x = x_23)[name = tensor("op_990")]; - tensor var_993_begin_0 = const()[name = tensor("op_993_begin_0"), val = tensor([0, 1, 0])]; - tensor var_993_end_0 = const()[name = tensor("op_993_end_0"), val = tensor([1, 16, 256])]; - tensor var_993_end_mask_0 = const()[name = tensor("op_993_end_mask_0"), val = tensor([true, true, true])]; - tensor var_993 = slice_by_index(begin = var_993_begin_0, end = var_993_end_0, end_mask = var_993_end_mask_0, x = window_5)[name = tensor("op_993")]; + tensor var_776_begin_0 = const()[name = tensor("op_776_begin_0"), val = tensor([0, 0, 0])]; + tensor var_776_end_0 = const()[name = tensor("op_776_end_0"), val = tensor([1, 1, 256])]; + tensor var_776_end_mask_0 = const()[name = tensor("op_776_end_mask_0"), val = tensor([true, false, true])]; + tensor var_776 = slice_by_index(begin = var_776_begin_0, end = var_776_end_0, end_mask = var_776_end_mask_0, x = x_15)[name = tensor("op_776")]; + tensor var_779_begin_0 = const()[name = tensor("op_779_begin_0"), val = tensor([0, 1, 0])]; + tensor var_779_end_0 = const()[name = tensor("op_779_end_0"), val = tensor([1, 16, 256])]; + tensor var_779_end_mask_0 = const()[name = tensor("op_779_end_mask_0"), val = tensor([true, true, true])]; + tensor var_779 = slice_by_index(begin = var_779_begin_0, end = var_779_end_0, end_mask = var_779_end_mask_0, x = window_17)[name = tensor("op_779")]; tensor rolled_13_interleave_0 = const()[name = tensor("rolled_13_interleave_0"), val = tensor(false)]; - tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_993, var_990))[name = tensor("rolled_13")]; + tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_779, var_776))[name = tensor("rolled_13")]; tensor clip_12 = clip(alpha = var_21, beta = var_20, x = filled_13)[name = tensor("clip_12")]; - tensor var_998 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_998")]; - tensor var_999_dtype_0 = const()[name = tensor("op_999_dtype_0"), val = tensor("fp32")]; - tensor var_1000 = const()[name = tensor("op_1000"), val = tensor([1, 16, 1])]; - tensor var_999 = cast(dtype = var_999_dtype_0, x = var_998)[name = tensor("cast_147")]; - tensor fill_mask_13 = reshape(shape = var_1000, x = var_999)[name = tensor("fill_mask_13")]; + tensor var_784 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_784")]; + tensor var_785_dtype_0 = const()[name = tensor("op_785_dtype_0"), val = tensor("fp32")]; + tensor var_786 = const()[name = tensor("op_786"), val = tensor([1, 16, 1])]; + tensor var_785 = cast(dtype = var_785_dtype_0, x = var_784)[name = tensor("cast_147")]; + tensor fill_mask_13 = reshape(shape = var_786, x = var_785)[name = tensor("fill_mask_13")]; tensor x_expanded_13_reps_0 = const()[name = tensor("x_expanded_13_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_990)[name = tensor("x_expanded_13")]; - tensor var_1004 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_1004")]; - tensor var_1005 = mul(x = window_5, y = var_1004)[name = tensor("op_1005")]; - tensor var_1006 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_1006")]; - tensor filled_window_13 = add(x = var_1005, y = var_1006)[name = tensor("filled_window_13")]; - tensor var_1008 = greater_equal(x = filled_13, y = var_16)[name = tensor("op_1008")]; - tensor var_1009_dtype_0 = const()[name = tensor("op_1009_dtype_0"), val = tensor("fp32")]; - tensor var_1010 = const()[name = tensor("op_1010"), val = tensor([1, 1, 1])]; - tensor var_1009 = cast(dtype = var_1009_dtype_0, x = var_1008)[name = tensor("cast_146")]; - tensor use_roll_13 = reshape(shape = var_1010, x = var_1009)[name = tensor("use_roll_13")]; - tensor var_1012 = sub(x = var_17, y = use_roll_13)[name = tensor("op_1012")]; - tensor var_1013 = mul(x = filled_window_13, y = var_1012)[name = tensor("op_1013")]; - tensor var_1014 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_1014")]; - tensor input_141 = add(x = var_1013, y = var_1014)[name = tensor("input_141")]; - tensor var_1016 = const()[name = tensor("op_1016"), val = tensor(0x1p+0)]; - tensor var_1017 = add(x = filled_13, y = var_1016)[name = tensor("op_1017")]; - tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_1017)[name = tensor("clip_13")]; - tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; - tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_141)[name = tensor("x_25")]; - tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_47_pad_type_0 = const()[name = tensor("inputs_47_pad_type_0"), val = tensor("valid")]; - tensor inputs_47_strides_0 = const()[name = tensor("inputs_47_strides_0"), val = tensor([1])]; - tensor inputs_47_pad_0 = const()[name = tensor("inputs_47_pad_0"), val = tensor([0, 0])]; - tensor inputs_47_dilations_0 = const()[name = tensor("inputs_47_dilations_0"), val = tensor([1])]; - tensor inputs_47_groups_0 = const()[name = tensor("inputs_47_groups_0"), val = tensor(1)]; - tensor input_143 = transpose(perm = input_143_perm_0, x = x_25)[name = tensor("transpose_46")]; - tensor inputs_47 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_47_dilations_0, groups = inputs_47_groups_0, pad = inputs_47_pad_0, pad_type = inputs_47_pad_type_0, strides = inputs_47_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_47")]; - tensor var_1038_split_sizes_0 = const()[name = tensor("op_1038_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1038_axis_0 = const()[name = tensor("op_1038_axis_0"), val = tensor(1)]; - tensor var_1038_0, tensor var_1038_1 = split(axis = var_1038_axis_0, split_sizes = var_1038_split_sizes_0, x = inputs_47)[name = tensor("op_1038")]; - tensor var_1040 = sigmoid(x = var_1038_1)[name = tensor("op_1040")]; - tensor inputs_49 = mul(x = var_1038_0, y = var_1040)[name = tensor("inputs_49")]; - tensor outputs_aug_13_pad_type_0 = const()[name = tensor("outputs_aug_13_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_13_pad_0 = const()[name = tensor("outputs_aug_13_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_13_groups_0 = const()[name = tensor("outputs_aug_13_groups_0"), val = tensor(256)]; - tensor outputs_aug_13_strides_0 = const()[name = tensor("outputs_aug_13_strides_0"), val = tensor([1])]; - tensor outputs_aug_13_dilations_0 = const()[name = tensor("outputs_aug_13_dilations_0"), val = tensor([1])]; - tensor outputs_aug_13 = conv(dilations = outputs_aug_13_dilations_0, groups = outputs_aug_13_groups_0, pad = outputs_aug_13_pad_0, pad_type = outputs_aug_13_pad_type_0, strides = outputs_aug_13_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_49)[name = tensor("outputs_aug_13")]; - tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; - tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([1, 256, 16])]; - tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; - tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug_13)[name = tensor("input_145")]; - tensor inputs_51 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_145)[name = tensor("inputs_51")]; - tensor input_147 = silu(x = inputs_51)[name = tensor("input_147")]; - tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; - tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; - tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; - tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; - tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; - tensor input_149 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; - tensor conv_out_13_perm_0 = const()[name = tensor("conv_out_13_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1071_begin_0 = const()[name = tensor("op_1071_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1071_end_0 = const()[name = tensor("op_1071_end_0"), val = tensor([1, 16, 256])]; - tensor var_1071_end_mask_0 = const()[name = tensor("op_1071_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_13 = transpose(perm = conv_out_13_perm_0, x = input_149)[name = tensor("transpose_45")]; - tensor var_1071 = slice_by_index(begin = var_1071_begin_0, end = var_1071_end_0, end_mask = var_1071_end_mask_0, x = conv_out_13)[name = tensor("op_1071")]; - tensor var_1074_begin_0 = const()[name = tensor("op_1074_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1074_end_0 = const()[name = tensor("op_1074_end_0"), val = tensor([1, 2, 256])]; - tensor var_1074_end_mask_0 = const()[name = tensor("op_1074_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1074 = slice_by_index(begin = var_1074_begin_0, end = var_1074_end_0, end_mask = var_1074_end_mask_0, x = x_23)[name = tensor("op_1074")]; - tensor var_1077_begin_0 = const()[name = tensor("op_1077_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1077_end_0 = const()[name = tensor("op_1077_end_0"), val = tensor([1, 16, 256])]; - tensor var_1077_end_mask_0 = const()[name = tensor("op_1077_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1077 = slice_by_index(begin = var_1077_begin_0, end = var_1077_end_0, end_mask = var_1077_end_mask_0, x = input_141)[name = tensor("op_1077")]; + tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_776)[name = tensor("x_expanded_13")]; + tensor var_790 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_790")]; + tensor var_791 = mul(x = window_17, y = var_790)[name = tensor("op_791")]; + tensor var_792 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_792")]; + tensor filled_window_13 = add(x = var_791, y = var_792)[name = tensor("filled_window_13")]; + tensor var_794 = greater_equal(x = filled_13, y = var_16)[name = tensor("op_794")]; + tensor var_795_dtype_0 = const()[name = tensor("op_795_dtype_0"), val = tensor("fp32")]; + tensor var_796 = const()[name = tensor("op_796"), val = tensor([1, 1, 1])]; + tensor var_795 = cast(dtype = var_795_dtype_0, x = var_794)[name = tensor("cast_146")]; + tensor use_roll_13 = reshape(shape = var_796, x = var_795)[name = tensor("use_roll_13")]; + tensor var_798 = sub(x = var_17, y = use_roll_13)[name = tensor("op_798")]; + tensor var_799 = mul(x = filled_window_13, y = var_798)[name = tensor("op_799")]; + tensor var_800 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_800")]; + tensor window_19 = add(x = var_799, y = var_800)[name = tensor("window_19")]; + tensor var_802 = const()[name = tensor("op_802"), val = tensor(0x1p+0)]; + tensor var_803 = add(x = filled_13, y = var_802)[name = tensor("op_803")]; + tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_803)[name = tensor("clip_13")]; + tensor var_806_begin_0 = const()[name = tensor("op_806_begin_0"), val = tensor([0, 1, 0])]; + tensor var_806_end_0 = const()[name = tensor("op_806_end_0"), val = tensor([1, 2, 256])]; + tensor var_806_end_mask_0 = const()[name = tensor("op_806_end_mask_0"), val = tensor([true, false, true])]; + tensor var_806 = slice_by_index(begin = var_806_begin_0, end = var_806_end_0, end_mask = var_806_end_mask_0, x = x_15)[name = tensor("op_806")]; + tensor var_809_begin_0 = const()[name = tensor("op_809_begin_0"), val = tensor([0, 1, 0])]; + tensor var_809_end_0 = const()[name = tensor("op_809_end_0"), val = tensor([1, 16, 256])]; + tensor var_809_end_mask_0 = const()[name = tensor("op_809_end_mask_0"), val = tensor([true, true, true])]; + tensor var_809 = slice_by_index(begin = var_809_begin_0, end = var_809_end_0, end_mask = var_809_end_mask_0, x = window_19)[name = tensor("op_809")]; tensor rolled_15_interleave_0 = const()[name = tensor("rolled_15_interleave_0"), val = tensor(false)]; - tensor rolled_15 = concat(axis = var_33, interleave = rolled_15_interleave_0, values = (var_1077, var_1074))[name = tensor("rolled_15")]; + tensor rolled_15 = concat(axis = var_33, interleave = rolled_15_interleave_0, values = (var_809, var_806))[name = tensor("rolled_15")]; tensor clip_14 = clip(alpha = var_21, beta = var_20, x = clip_13)[name = tensor("clip_14")]; - tensor var_1082 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_1082")]; - tensor var_1083_dtype_0 = const()[name = tensor("op_1083_dtype_0"), val = tensor("fp32")]; - tensor var_1084 = const()[name = tensor("op_1084"), val = tensor([1, 16, 1])]; - tensor var_1083 = cast(dtype = var_1083_dtype_0, x = var_1082)[name = tensor("cast_145")]; - tensor fill_mask_15 = reshape(shape = var_1084, x = var_1083)[name = tensor("fill_mask_15")]; + tensor var_814 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_814")]; + tensor var_815_dtype_0 = const()[name = tensor("op_815_dtype_0"), val = tensor("fp32")]; + tensor var_816 = const()[name = tensor("op_816"), val = tensor([1, 16, 1])]; + tensor var_815 = cast(dtype = var_815_dtype_0, x = var_814)[name = tensor("cast_145")]; + tensor fill_mask_15 = reshape(shape = var_816, x = var_815)[name = tensor("fill_mask_15")]; tensor x_expanded_15_reps_0 = const()[name = tensor("x_expanded_15_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_1074)[name = tensor("x_expanded_15")]; - tensor var_1088 = sub(x = var_17, y = fill_mask_15)[name = tensor("op_1088")]; - tensor var_1089 = mul(x = input_141, y = var_1088)[name = tensor("op_1089")]; - tensor var_1090 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_1090")]; - tensor filled_window_15 = add(x = var_1089, y = var_1090)[name = tensor("filled_window_15")]; - tensor var_1092 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_1092")]; - tensor var_1093_dtype_0 = const()[name = tensor("op_1093_dtype_0"), val = tensor("fp32")]; - tensor var_1094 = const()[name = tensor("op_1094"), val = tensor([1, 1, 1])]; - tensor var_1093 = cast(dtype = var_1093_dtype_0, x = var_1092)[name = tensor("cast_144")]; - tensor use_roll_15 = reshape(shape = var_1094, x = var_1093)[name = tensor("use_roll_15")]; - tensor var_1096 = sub(x = var_17, y = use_roll_15)[name = tensor("op_1096")]; - tensor var_1097 = mul(x = filled_window_15, y = var_1096)[name = tensor("op_1097")]; - tensor var_1098 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_1098")]; - tensor input_151 = add(x = var_1097, y = var_1098)[name = tensor("input_151")]; - tensor var_1100 = const()[name = tensor("op_1100"), val = tensor(0x1p+0)]; - tensor var_1101 = add(x = clip_13, y = var_1100)[name = tensor("op_1101")]; - tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_1101)[name = tensor("clip_15")]; - tensor x_27_axes_0 = const()[name = tensor("x_27_axes_0"), val = tensor([-1])]; - tensor x_27 = layer_norm(axes = x_27_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_151)[name = tensor("x_27")]; - tensor input_153_perm_0 = const()[name = tensor("input_153_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_53_pad_type_0 = const()[name = tensor("inputs_53_pad_type_0"), val = tensor("valid")]; - tensor inputs_53_strides_0 = const()[name = tensor("inputs_53_strides_0"), val = tensor([1])]; - tensor inputs_53_pad_0 = const()[name = tensor("inputs_53_pad_0"), val = tensor([0, 0])]; - tensor inputs_53_dilations_0 = const()[name = tensor("inputs_53_dilations_0"), val = tensor([1])]; - tensor inputs_53_groups_0 = const()[name = tensor("inputs_53_groups_0"), val = tensor(1)]; - tensor input_153 = transpose(perm = input_153_perm_0, x = x_27)[name = tensor("transpose_44")]; - tensor inputs_53 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_53_dilations_0, groups = inputs_53_groups_0, pad = inputs_53_pad_0, pad_type = inputs_53_pad_type_0, strides = inputs_53_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_153)[name = tensor("inputs_53")]; - tensor var_1122_split_sizes_0 = const()[name = tensor("op_1122_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1122_axis_0 = const()[name = tensor("op_1122_axis_0"), val = tensor(1)]; - tensor var_1122_0, tensor var_1122_1 = split(axis = var_1122_axis_0, split_sizes = var_1122_split_sizes_0, x = inputs_53)[name = tensor("op_1122")]; - tensor var_1124 = sigmoid(x = var_1122_1)[name = tensor("op_1124")]; - tensor inputs_55 = mul(x = var_1122_0, y = var_1124)[name = tensor("inputs_55")]; - tensor outputs_aug_15_pad_type_0 = const()[name = tensor("outputs_aug_15_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_15_pad_0 = const()[name = tensor("outputs_aug_15_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_15_groups_0 = const()[name = tensor("outputs_aug_15_groups_0"), val = tensor(256)]; - tensor outputs_aug_15_strides_0 = const()[name = tensor("outputs_aug_15_strides_0"), val = tensor([1])]; - tensor outputs_aug_15_dilations_0 = const()[name = tensor("outputs_aug_15_dilations_0"), val = tensor([1])]; - tensor outputs_aug_15 = conv(dilations = outputs_aug_15_dilations_0, groups = outputs_aug_15_groups_0, pad = outputs_aug_15_pad_0, pad_type = outputs_aug_15_pad_type_0, strides = outputs_aug_15_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_55)[name = tensor("outputs_aug_15")]; - tensor input_155_begin_0 = const()[name = tensor("input_155_begin_0"), val = tensor([0, 0, 0])]; - tensor input_155_end_0 = const()[name = tensor("input_155_end_0"), val = tensor([1, 256, 16])]; - tensor input_155_end_mask_0 = const()[name = tensor("input_155_end_mask_0"), val = tensor([true, true, false])]; - tensor input_155 = slice_by_index(begin = input_155_begin_0, end = input_155_end_0, end_mask = input_155_end_mask_0, x = outputs_aug_15)[name = tensor("input_155")]; - tensor inputs_57 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_155)[name = tensor("inputs_57")]; - tensor input_157 = silu(x = inputs_57)[name = tensor("input_157")]; - tensor input_159_pad_type_0 = const()[name = tensor("input_159_pad_type_0"), val = tensor("valid")]; - tensor input_159_strides_0 = const()[name = tensor("input_159_strides_0"), val = tensor([1])]; - tensor input_159_pad_0 = const()[name = tensor("input_159_pad_0"), val = tensor([0, 0])]; - tensor input_159_dilations_0 = const()[name = tensor("input_159_dilations_0"), val = tensor([1])]; - tensor input_159_groups_0 = const()[name = tensor("input_159_groups_0"), val = tensor(1)]; - tensor input_159 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_159_dilations_0, groups = input_159_groups_0, pad = input_159_pad_0, pad_type = input_159_pad_type_0, strides = input_159_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_157)[name = tensor("input_159")]; - tensor conv_out_15_perm_0 = const()[name = tensor("conv_out_15_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1155_begin_0 = const()[name = tensor("op_1155_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1155_end_0 = const()[name = tensor("op_1155_end_0"), val = tensor([1, 16, 256])]; - tensor var_1155_end_mask_0 = const()[name = tensor("op_1155_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_15 = transpose(perm = conv_out_15_perm_0, x = input_159)[name = tensor("transpose_43")]; - tensor var_1155 = slice_by_index(begin = var_1155_begin_0, end = var_1155_end_0, end_mask = var_1155_end_mask_0, x = conv_out_15)[name = tensor("op_1155")]; - tensor var_1158_begin_0 = const()[name = tensor("op_1158_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1158_end_0 = const()[name = tensor("op_1158_end_0"), val = tensor([1, 1, 256])]; - tensor var_1158_end_mask_0 = const()[name = tensor("op_1158_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1158 = slice_by_index(begin = var_1158_begin_0, end = var_1158_end_0, end_mask = var_1158_end_mask_0, x = x_23)[name = tensor("op_1158")]; - tensor var_1161_begin_0 = const()[name = tensor("op_1161_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1161_end_0 = const()[name = tensor("op_1161_end_0"), val = tensor([1, 16, 256])]; - tensor var_1161_end_mask_0 = const()[name = tensor("op_1161_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1161 = slice_by_index(begin = var_1161_begin_0, end = var_1161_end_0, end_mask = var_1161_end_mask_0, x = input_151)[name = tensor("op_1161")]; + tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_806)[name = tensor("x_expanded_15")]; + tensor var_820 = sub(x = var_17, y = fill_mask_15)[name = tensor("op_820")]; + tensor var_821 = mul(x = window_19, y = var_820)[name = tensor("op_821")]; + tensor var_822 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_822")]; + tensor filled_window_15 = add(x = var_821, y = var_822)[name = tensor("filled_window_15")]; + tensor var_824 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_824")]; + tensor var_825_dtype_0 = const()[name = tensor("op_825_dtype_0"), val = tensor("fp32")]; + tensor var_826 = const()[name = tensor("op_826"), val = tensor([1, 1, 1])]; + tensor var_825 = cast(dtype = var_825_dtype_0, x = var_824)[name = tensor("cast_144")]; + tensor use_roll_15 = reshape(shape = var_826, x = var_825)[name = tensor("use_roll_15")]; + tensor var_828 = sub(x = var_17, y = use_roll_15)[name = tensor("op_828")]; + tensor var_829 = mul(x = filled_window_15, y = var_828)[name = tensor("op_829")]; + tensor var_830 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_830")]; + tensor window_21 = add(x = var_829, y = var_830)[name = tensor("window_21")]; + tensor var_832 = const()[name = tensor("op_832"), val = tensor(0x1p+0)]; + tensor var_833 = add(x = clip_13, y = var_832)[name = tensor("op_833")]; + tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_833)[name = tensor("clip_15")]; + tensor var_836_begin_0 = const()[name = tensor("op_836_begin_0"), val = tensor([0, 2, 0])]; + tensor var_836_end_0 = const()[name = tensor("op_836_end_0"), val = tensor([1, 1, 256])]; + tensor var_836_end_mask_0 = const()[name = tensor("op_836_end_mask_0"), val = tensor([true, true, true])]; + tensor var_836 = slice_by_index(begin = var_836_begin_0, end = var_836_end_0, end_mask = var_836_end_mask_0, x = x_15)[name = tensor("op_836")]; + tensor var_839_begin_0 = const()[name = tensor("op_839_begin_0"), val = tensor([0, 1, 0])]; + tensor var_839_end_0 = const()[name = tensor("op_839_end_0"), val = tensor([1, 16, 256])]; + tensor var_839_end_mask_0 = const()[name = tensor("op_839_end_mask_0"), val = tensor([true, true, true])]; + tensor var_839 = slice_by_index(begin = var_839_begin_0, end = var_839_end_0, end_mask = var_839_end_mask_0, x = window_21)[name = tensor("op_839")]; tensor rolled_17_interleave_0 = const()[name = tensor("rolled_17_interleave_0"), val = tensor(false)]; - tensor rolled_17 = concat(axis = var_33, interleave = rolled_17_interleave_0, values = (var_1161, var_1158))[name = tensor("rolled_17")]; + tensor rolled_17 = concat(axis = var_33, interleave = rolled_17_interleave_0, values = (var_839, var_836))[name = tensor("rolled_17")]; tensor clip_16 = clip(alpha = var_21, beta = var_20, x = clip_15)[name = tensor("clip_16")]; - tensor var_1166 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_1166")]; - tensor var_1167_dtype_0 = const()[name = tensor("op_1167_dtype_0"), val = tensor("fp32")]; - tensor var_1168 = const()[name = tensor("op_1168"), val = tensor([1, 16, 1])]; - tensor var_1167 = cast(dtype = var_1167_dtype_0, x = var_1166)[name = tensor("cast_143")]; - tensor fill_mask_17 = reshape(shape = var_1168, x = var_1167)[name = tensor("fill_mask_17")]; + tensor var_844 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_844")]; + tensor var_845_dtype_0 = const()[name = tensor("op_845_dtype_0"), val = tensor("fp32")]; + tensor var_846 = const()[name = tensor("op_846"), val = tensor([1, 16, 1])]; + tensor var_845 = cast(dtype = var_845_dtype_0, x = var_844)[name = tensor("cast_143")]; + tensor fill_mask_17 = reshape(shape = var_846, x = var_845)[name = tensor("fill_mask_17")]; tensor x_expanded_17_reps_0 = const()[name = tensor("x_expanded_17_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_1158)[name = tensor("x_expanded_17")]; - tensor var_1172 = sub(x = var_17, y = fill_mask_17)[name = tensor("op_1172")]; - tensor var_1173 = mul(x = input_151, y = var_1172)[name = tensor("op_1173")]; - tensor var_1174 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_1174")]; - tensor filled_window_17 = add(x = var_1173, y = var_1174)[name = tensor("filled_window_17")]; - tensor var_1176 = greater_equal(x = clip_15, y = var_16)[name = tensor("op_1176")]; - tensor var_1177_dtype_0 = const()[name = tensor("op_1177_dtype_0"), val = tensor("fp32")]; - tensor var_1178 = const()[name = tensor("op_1178"), val = tensor([1, 1, 1])]; - tensor var_1177 = cast(dtype = var_1177_dtype_0, x = var_1176)[name = tensor("cast_142")]; - tensor use_roll_17 = reshape(shape = var_1178, x = var_1177)[name = tensor("use_roll_17")]; - tensor var_1180 = sub(x = var_17, y = use_roll_17)[name = tensor("op_1180")]; - tensor var_1181 = mul(x = filled_window_17, y = var_1180)[name = tensor("op_1181")]; - tensor var_1182 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_1182")]; - tensor input_161 = add(x = var_1181, y = var_1182)[name = tensor("input_161")]; - tensor var_1184 = const()[name = tensor("op_1184"), val = tensor(0x1p+0)]; - tensor var_1185 = add(x = clip_15, y = var_1184)[name = tensor("op_1185")]; - tensor clip_17 = clip(alpha = var_21, beta = var_16, x = var_1185)[name = tensor("clip_17")]; - tensor x_29_axes_0 = const()[name = tensor("x_29_axes_0"), val = tensor([-1])]; - tensor x_29 = layer_norm(axes = x_29_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_161)[name = tensor("x_29")]; - tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_59_pad_type_0 = const()[name = tensor("inputs_59_pad_type_0"), val = tensor("valid")]; - tensor inputs_59_strides_0 = const()[name = tensor("inputs_59_strides_0"), val = tensor([1])]; - tensor inputs_59_pad_0 = const()[name = tensor("inputs_59_pad_0"), val = tensor([0, 0])]; - tensor inputs_59_dilations_0 = const()[name = tensor("inputs_59_dilations_0"), val = tensor([1])]; - tensor inputs_59_groups_0 = const()[name = tensor("inputs_59_groups_0"), val = tensor(1)]; - tensor input_163 = transpose(perm = input_163_perm_0, x = x_29)[name = tensor("transpose_42")]; - tensor inputs_59 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_59_dilations_0, groups = inputs_59_groups_0, pad = inputs_59_pad_0, pad_type = inputs_59_pad_type_0, strides = inputs_59_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_163)[name = tensor("inputs_59")]; - tensor var_1206_split_sizes_0 = const()[name = tensor("op_1206_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1206_axis_0 = const()[name = tensor("op_1206_axis_0"), val = tensor(1)]; - tensor var_1206_0, tensor var_1206_1 = split(axis = var_1206_axis_0, split_sizes = var_1206_split_sizes_0, x = inputs_59)[name = tensor("op_1206")]; - tensor var_1208 = sigmoid(x = var_1206_1)[name = tensor("op_1208")]; - tensor inputs_61 = mul(x = var_1206_0, y = var_1208)[name = tensor("inputs_61")]; - tensor outputs_aug_17_pad_type_0 = const()[name = tensor("outputs_aug_17_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_17_pad_0 = const()[name = tensor("outputs_aug_17_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_17_groups_0 = const()[name = tensor("outputs_aug_17_groups_0"), val = tensor(256)]; - tensor outputs_aug_17_strides_0 = const()[name = tensor("outputs_aug_17_strides_0"), val = tensor([1])]; - tensor outputs_aug_17_dilations_0 = const()[name = tensor("outputs_aug_17_dilations_0"), val = tensor([1])]; - tensor outputs_aug_17 = conv(dilations = outputs_aug_17_dilations_0, groups = outputs_aug_17_groups_0, pad = outputs_aug_17_pad_0, pad_type = outputs_aug_17_pad_type_0, strides = outputs_aug_17_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_61)[name = tensor("outputs_aug_17")]; - tensor input_165_begin_0 = const()[name = tensor("input_165_begin_0"), val = tensor([0, 0, 0])]; - tensor input_165_end_0 = const()[name = tensor("input_165_end_0"), val = tensor([1, 256, 16])]; - tensor input_165_end_mask_0 = const()[name = tensor("input_165_end_mask_0"), val = tensor([true, true, false])]; - tensor input_165 = slice_by_index(begin = input_165_begin_0, end = input_165_end_0, end_mask = input_165_end_mask_0, x = outputs_aug_17)[name = tensor("input_165")]; - tensor inputs_63 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_165)[name = tensor("inputs_63")]; - tensor input_167 = silu(x = inputs_63)[name = tensor("input_167")]; - tensor input_169_pad_type_0 = const()[name = tensor("input_169_pad_type_0"), val = tensor("valid")]; - tensor input_169_strides_0 = const()[name = tensor("input_169_strides_0"), val = tensor([1])]; - tensor input_169_pad_0 = const()[name = tensor("input_169_pad_0"), val = tensor([0, 0])]; - tensor input_169_dilations_0 = const()[name = tensor("input_169_dilations_0"), val = tensor([1])]; - tensor input_169_groups_0 = const()[name = tensor("input_169_groups_0"), val = tensor(1)]; - tensor input_169 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_169_dilations_0, groups = input_169_groups_0, pad = input_169_pad_0, pad_type = input_169_pad_type_0, strides = input_169_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_167)[name = tensor("input_169")]; - tensor conv_out_17_perm_0 = const()[name = tensor("conv_out_17_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1239_begin_0 = const()[name = tensor("op_1239_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1239_end_0 = const()[name = tensor("op_1239_end_0"), val = tensor([1, 16, 256])]; - tensor var_1239_end_mask_0 = const()[name = tensor("op_1239_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_17 = transpose(perm = conv_out_17_perm_0, x = input_169)[name = tensor("transpose_41")]; - tensor var_1239 = slice_by_index(begin = var_1239_begin_0, end = var_1239_end_0, end_mask = var_1239_end_mask_0, x = conv_out_17)[name = tensor("op_1239")]; - tensor var_1242_interleave_0 = const()[name = tensor("op_1242_interleave_0"), val = tensor(false)]; - tensor var_1242 = concat(axis = var_33, interleave = var_1242_interleave_0, values = (var_1071, var_1155, var_1239))[name = tensor("op_1242")]; - tensor input_171 = add(x = x_23, y = var_1242)[name = tensor("input_171")]; - tensor input_173_axes_0 = const()[name = tensor("input_173_axes_0"), val = tensor([-1])]; - tensor input_173 = layer_norm(axes = input_173_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_171)[name = tensor("input_173")]; - tensor inputs_65 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_173)[name = tensor("linear_26")]; - tensor input_175 = silu(x = inputs_65)[name = tensor("input_175")]; - tensor input_179 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_175)[name = tensor("linear_27")]; - tensor var_1265 = const()[name = tensor("op_1265"), val = tensor(0x1p-1)]; - tensor var_1266 = mul(x = input_179, y = var_1265)[name = tensor("op_1266")]; - tensor input_181 = add(x = var_1266, y = input_171)[name = tensor("input_181")]; - tensor input_183_axes_0 = const()[name = tensor("input_183_axes_0"), val = tensor([-1])]; - tensor input_183 = layer_norm(axes = input_183_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_181)[name = tensor("input_183")]; - tensor input_185_axes_0 = const()[name = tensor("input_185_axes_0"), val = tensor([-1])]; - tensor input_185 = layer_norm(axes = input_185_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_183)[name = tensor("input_185")]; - tensor inputs_67 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_185)[name = tensor("linear_28")]; - tensor input_187 = silu(x = inputs_67)[name = tensor("input_187")]; - tensor input_191 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_187)[name = tensor("linear_29")]; - tensor var_1295 = const()[name = tensor("op_1295"), val = tensor(0x1p-1)]; - tensor var_1296 = mul(x = input_191, y = var_1295)[name = tensor("op_1296")]; - tensor input_193 = add(x = var_1296, y = input_183)[name = tensor("input_193")]; - tensor x_31_axes_0 = const()[name = tensor("x_31_axes_0"), val = tensor([-1])]; - tensor x_31 = layer_norm(axes = x_31_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_193)[name = tensor("x_31")]; + tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_836)[name = tensor("x_expanded_17")]; + tensor var_850 = sub(x = var_17, y = fill_mask_17)[name = tensor("op_850")]; + tensor var_851 = mul(x = window_21, y = var_850)[name = tensor("op_851")]; + tensor var_852 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_852")]; + tensor filled_window_17 = add(x = var_851, y = var_852)[name = tensor("filled_window_17")]; + tensor var_854 = greater_equal(x = clip_15, y = var_16)[name = tensor("op_854")]; + tensor var_855_dtype_0 = const()[name = tensor("op_855_dtype_0"), val = tensor("fp32")]; + tensor var_856 = const()[name = tensor("op_856"), val = tensor([1, 1, 1])]; + tensor var_855 = cast(dtype = var_855_dtype_0, x = var_854)[name = tensor("cast_142")]; + tensor use_roll_17 = reshape(shape = var_856, x = var_855)[name = tensor("use_roll_17")]; + tensor var_858 = sub(x = var_17, y = use_roll_17)[name = tensor("op_858")]; + tensor var_859 = mul(x = filled_window_17, y = var_858)[name = tensor("op_859")]; + tensor var_860 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_860")]; + tensor window_23 = add(x = var_859, y = var_860)[name = tensor("window_23")]; + tensor var_862 = const()[name = tensor("op_862"), val = tensor(0x1p+0)]; + tensor var_863 = add(x = clip_15, y = var_862)[name = tensor("op_863")]; + tensor clip_17 = clip(alpha = var_21, beta = var_16, x = var_863)[name = tensor("clip_17")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_30, interleave = input_101_interleave_0, values = (window_19, window_21, window_23))[name = tensor("input_101")]; + tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; + tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; + tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_23_pad_type_0 = const()[name = tensor("inputs_23_pad_type_0"), val = tensor("valid")]; + tensor inputs_23_strides_0 = const()[name = tensor("inputs_23_strides_0"), val = tensor([1])]; + tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; + tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; + tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; + tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; + tensor var_886_split_sizes_0 = const()[name = tensor("op_886_split_sizes_0"), val = tensor([256, 256])]; + tensor var_886_axis_0 = const()[name = tensor("op_886_axis_0"), val = tensor(1)]; + tensor var_886_0, tensor var_886_1 = split(axis = var_886_axis_0, split_sizes = var_886_split_sizes_0, x = inputs_23)[name = tensor("op_886")]; + tensor var_888 = sigmoid(x = var_886_1)[name = tensor("op_888")]; + tensor inputs_25 = mul(x = var_886_0, y = var_888)[name = tensor("inputs_25")]; + tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; + tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; + tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; + tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_25)[name = tensor("outputs_aug_5")]; + tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; + tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([3, 256, 16])]; + tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; + tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_5)[name = tensor("input_105")]; + tensor inputs_27 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_105)[name = tensor("inputs_27")]; + tensor input_107 = silu(x = inputs_27)[name = tensor("input_107")]; + tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; + tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; + tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; + tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; + tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; + tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; + tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; + tensor var_919_begin_0 = const()[name = tensor("op_919_begin_0"), val = tensor([0, -1, 0])]; + tensor var_919_end_0 = const()[name = tensor("op_919_end_0"), val = tensor([3, 16, 256])]; + tensor var_919_end_mask_0 = const()[name = tensor("op_919_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_919 = slice_by_index(begin = var_919_begin_0, end = var_919_end_0, end_mask = var_919_end_mask_0, x = conv_out_5)[name = tensor("op_919")]; + tensor var_921_perm_0 = const()[name = tensor("op_921_perm_0"), val = tensor([1, 0, 2])]; + tensor var_921 = transpose(perm = var_921_perm_0, x = var_919)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_921)[name = tensor("input_111")]; + tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; + tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; + tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; + tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; + tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; + tensor var_944 = const()[name = tensor("op_944"), val = tensor(0x1p-1)]; + tensor var_945 = mul(x = input_119, y = var_944)[name = tensor("op_945")]; + tensor input_121 = add(x = var_945, y = input_111)[name = tensor("input_121")]; + tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; + tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; + tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; + tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; + tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; + tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; + tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; + tensor var_974 = const()[name = tensor("op_974"), val = tensor(0x1p-1)]; + tensor var_975 = mul(x = input_131, y = var_974)[name = tensor("op_975")]; + tensor input_133 = add(x = var_975, y = input_123)[name = tensor("input_133")]; + tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; + tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; tensor prev_kv_7_end_0 = const()[name = tensor("prev_kv_7_end_0"), val = tensor([4, 1, 4, 64, 64])]; tensor prev_kv_7_end_mask_0 = const()[name = tensor("prev_kv_7_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1203,309 +976,233 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_1310 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_31)[name = tensor("linear_30")]; - tensor var_1311 = const()[name = tensor("op_1311"), val = tensor([1, 3, 4, 64])]; - tensor var_1312 = reshape(shape = var_1311, x = var_1310)[name = tensor("op_1312")]; + tensor var_989 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_990 = const()[name = tensor("op_990"), val = tensor([1, 3, 4, 64])]; + tensor var_991 = reshape(shape = var_990, x = var_989)[name = tensor("op_991")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1316 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_31)[name = tensor("linear_31")]; - tensor var_1317 = const()[name = tensor("op_1317"), val = tensor(0x1p-3)]; - tensor var_1318 = mul(x = var_1316, y = var_1317)[name = tensor("op_1318")]; - tensor var_1319 = const()[name = tensor("op_1319"), val = tensor([1, 3, 4, 64])]; - tensor var_1320 = reshape(shape = var_1319, x = var_1318)[name = tensor("op_1320")]; + tensor var_995 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_996 = const()[name = tensor("op_996"), val = tensor(0x1p-3)]; + tensor var_997 = mul(x = var_995, y = var_996)[name = tensor("op_997")]; + tensor var_998 = const()[name = tensor("op_998"), val = tensor([1, 3, 4, 64])]; + tensor var_999 = reshape(shape = var_998, x = var_997)[name = tensor("op_999")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1324 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_31)[name = tensor("linear_32")]; - tensor var_1325 = const()[name = tensor("op_1325"), val = tensor([1, 3, 4, 64])]; - tensor var_1326 = reshape(shape = var_1325, x = var_1324)[name = tensor("op_1326")]; + tensor var_1003 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_1004 = const()[name = tensor("op_1004"), val = tensor([1, 3, 4, 64])]; + tensor var_1005 = reshape(shape = var_1004, x = var_1003)[name = tensor("op_1005")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_197 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_31)[name = tensor("linear_33")]; + tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; tensor s_t_7 = add(x = prev_scale_7, y = encoder__t_index)[name = tensor("s_t_7")]; tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_1320)[name = tensor("transpose_39")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_1312)[name = tensor("transpose_40")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_999)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_991)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_1336 = const()[name = tensor("op_1336"), val = tensor([3, 1])]; - tensor var_1337 = reshape(shape = var_1336, x = sqrt_s_t_7)[name = tensor("op_1337")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_1337)[name = tensor("M_7")]; - tensor var_1339 = mul(x = qk_7, y = M_7)[name = tensor("op_1339")]; + tensor var_1015 = const()[name = tensor("op_1015"), val = tensor([3, 1])]; + tensor var_1016 = reshape(shape = var_1015, x = sqrt_s_t_7)[name = tensor("op_1016")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_1016)[name = tensor("M_7")]; + tensor var_1018 = mul(x = qk_7, y = M_7)[name = tensor("op_1018")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_1326)[name = tensor("transpose_38")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1339, y = v_7)[name = tensor("inner_7")]; - tensor var_1341_transpose_x_0 = const()[name = tensor("op_1341_transpose_x_0"), val = tensor(false)]; - tensor var_1341_transpose_y_0 = const()[name = tensor("op_1341_transpose_y_0"), val = tensor(false)]; - tensor var_1341 = matmul(transpose_x = var_1341_transpose_x_0, transpose_y = var_1341_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1341")]; - tensor var_1342 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1342")]; - tensor var_1343 = const()[name = tensor("op_1343"), val = tensor([1, 1, 3, 1])]; - tensor var_1344 = reshape(shape = var_1343, x = var_1342)[name = tensor("op_1344")]; - tensor cross_7 = mul(x = var_1341, y = var_1344)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_1005)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1018, y = v_7)[name = tensor("inner_7")]; + tensor var_1020_transpose_x_0 = const()[name = tensor("op_1020_transpose_x_0"), val = tensor(false)]; + tensor var_1020_transpose_y_0 = const()[name = tensor("op_1020_transpose_y_0"), val = tensor(false)]; + tensor var_1020 = matmul(transpose_x = var_1020_transpose_x_0, transpose_y = var_1020_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1020")]; + tensor var_1021 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1021")]; + tensor var_1022 = const()[name = tensor("op_1022"), val = tensor([1, 1, 3, 1])]; + tensor var_1023 = reshape(shape = var_1022, x = var_1021)[name = tensor("op_1023")]; + tensor cross_7 = mul(x = var_1020, y = var_1023)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_1347 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1347")]; - tensor var_1349_transpose_x_1 = const()[name = tensor("op_1349_transpose_x_1"), val = tensor(true)]; - tensor var_1349_transpose_y_1 = const()[name = tensor("op_1349_transpose_y_1"), val = tensor(false)]; - tensor var_1349 = matmul(transpose_x = var_1349_transpose_x_1, transpose_y = var_1349_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1349")]; - tensor new_kv_unnorm_7 = add(x = var_1347, y = var_1349)[name = tensor("new_kv_unnorm_7")]; - tensor var_1351 = const()[name = tensor("op_1351"), val = tensor(0x1.8p+1)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_1351)[name = tensor("new_scale_7")]; - tensor var_1353 = sqrt(x = new_scale_7)[name = tensor("op_1353")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1353)[name = tensor("nkv_1")]; - tensor var_1355_perm_0 = const()[name = tensor("op_1355_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1026 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1026")]; + tensor var_1028_transpose_x_1 = const()[name = tensor("op_1028_transpose_x_1"), val = tensor(true)]; + tensor var_1028_transpose_y_1 = const()[name = tensor("op_1028_transpose_y_1"), val = tensor(false)]; + tensor var_1028 = matmul(transpose_x = var_1028_transpose_x_1, transpose_y = var_1028_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1028")]; + tensor new_kv_unnorm_7 = add(x = var_1026, y = var_1028)[name = tensor("new_kv_unnorm_7")]; + tensor var_1030 = const()[name = tensor("op_1030"), val = tensor(0x1.8p+1)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_1030)[name = tensor("new_scale_7")]; + tensor var_1032 = sqrt(x = new_scale_7)[name = tensor("op_1032")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1032)[name = tensor("nkv_1")]; + tensor var_1034_perm_0 = const()[name = tensor("op_1034_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_1355 = transpose(perm = var_1355_perm_0, x = out_19)[name = tensor("transpose_37")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1355)[name = tensor("out_21")]; - tensor var_1359 = const()[name = tensor("op_1359"), val = tensor([1, 3, 256])]; - tensor out_23 = reshape(shape = var_1359, x = out_21)[name = tensor("out_23")]; - tensor var_1361 = silu(x = input_197)[name = tensor("op_1361")]; - tensor input_199 = mul(x = var_1361, y = out_23)[name = tensor("input_199")]; - tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_199)[name = tensor("linear_34")]; - tensor x_33 = add(x = input_193, y = ret_out_7)[name = tensor("x_33")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor var_1034 = transpose(perm = var_1034_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1034)[name = tensor("out_21")]; + tensor var_1038 = const()[name = tensor("op_1038"), val = tensor([1, 3, 256])]; + tensor out_23 = reshape(shape = var_1038, x = out_21)[name = tensor("out_23")]; + tensor var_1040 = silu(x = input_137)[name = tensor("op_1040")]; + tensor input_139 = mul(x = var_1040, y = out_23)[name = tensor("input_139")]; + tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; + tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; + tensor window_25_begin_0 = const()[name = tensor("window_25_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_25_end_0 = const()[name = tensor("window_25_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_25_end_mask_0 = const()[name = tensor("window_25_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_25_squeeze_mask_0 = const()[name = tensor("window_25_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_25 = slice_by_index(begin = window_25_begin_0, end = window_25_end_0, end_mask = window_25_end_mask_0, squeeze_mask = window_25_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_25")]; tensor filled_19_begin_0 = const()[name = tensor("filled_19_begin_0"), val = tensor([3, 0])]; tensor filled_19_end_0 = const()[name = tensor("filled_19_end_0"), val = tensor([4, 1])]; tensor filled_19_end_mask_0 = const()[name = tensor("filled_19_end_mask_0"), val = tensor([false, true])]; tensor filled_19_squeeze_mask_0 = const()[name = tensor("filled_19_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_19 = slice_by_index(begin = filled_19_begin_0, end = filled_19_end_0, end_mask = filled_19_end_mask_0, squeeze_mask = filled_19_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_19")]; - tensor var_1370_begin_0 = const()[name = tensor("op_1370_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1370_end_0 = const()[name = tensor("op_1370_end_0"), val = tensor([1, 1, 256])]; - tensor var_1370_end_mask_0 = const()[name = tensor("op_1370_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1370 = slice_by_index(begin = var_1370_begin_0, end = var_1370_end_0, end_mask = var_1370_end_mask_0, x = x_33)[name = tensor("op_1370")]; - tensor var_1373_begin_0 = const()[name = tensor("op_1373_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1373_end_0 = const()[name = tensor("op_1373_end_0"), val = tensor([1, 16, 256])]; - tensor var_1373_end_mask_0 = const()[name = tensor("op_1373_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1373 = slice_by_index(begin = var_1373_begin_0, end = var_1373_end_0, end_mask = var_1373_end_mask_0, x = window)[name = tensor("op_1373")]; + tensor var_1049_begin_0 = const()[name = tensor("op_1049_begin_0"), val = tensor([0, 0, 0])]; + tensor var_1049_end_0 = const()[name = tensor("op_1049_end_0"), val = tensor([1, 1, 256])]; + tensor var_1049_end_mask_0 = const()[name = tensor("op_1049_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1049 = slice_by_index(begin = var_1049_begin_0, end = var_1049_end_0, end_mask = var_1049_end_mask_0, x = x_21)[name = tensor("op_1049")]; + tensor var_1052_begin_0 = const()[name = tensor("op_1052_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1052_end_0 = const()[name = tensor("op_1052_end_0"), val = tensor([1, 16, 256])]; + tensor var_1052_end_mask_0 = const()[name = tensor("op_1052_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1052 = slice_by_index(begin = var_1052_begin_0, end = var_1052_end_0, end_mask = var_1052_end_mask_0, x = window_25)[name = tensor("op_1052")]; tensor rolled_19_interleave_0 = const()[name = tensor("rolled_19_interleave_0"), val = tensor(false)]; - tensor rolled_19 = concat(axis = var_33, interleave = rolled_19_interleave_0, values = (var_1373, var_1370))[name = tensor("rolled_19")]; + tensor rolled_19 = concat(axis = var_33, interleave = rolled_19_interleave_0, values = (var_1052, var_1049))[name = tensor("rolled_19")]; tensor clip_18 = clip(alpha = var_21, beta = var_20, x = filled_19)[name = tensor("clip_18")]; - tensor var_1378 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_1378")]; - tensor var_1379_dtype_0 = const()[name = tensor("op_1379_dtype_0"), val = tensor("fp32")]; - tensor var_1380 = const()[name = tensor("op_1380"), val = tensor([1, 16, 1])]; - tensor var_1379 = cast(dtype = var_1379_dtype_0, x = var_1378)[name = tensor("cast_141")]; - tensor fill_mask_19 = reshape(shape = var_1380, x = var_1379)[name = tensor("fill_mask_19")]; + tensor var_1057 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_1057")]; + tensor var_1058_dtype_0 = const()[name = tensor("op_1058_dtype_0"), val = tensor("fp32")]; + tensor var_1059 = const()[name = tensor("op_1059"), val = tensor([1, 16, 1])]; + tensor var_1058 = cast(dtype = var_1058_dtype_0, x = var_1057)[name = tensor("cast_141")]; + tensor fill_mask_19 = reshape(shape = var_1059, x = var_1058)[name = tensor("fill_mask_19")]; tensor x_expanded_19_reps_0 = const()[name = tensor("x_expanded_19_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_1370)[name = tensor("x_expanded_19")]; - tensor var_1384 = sub(x = var_17, y = fill_mask_19)[name = tensor("op_1384")]; - tensor var_1385 = mul(x = window, y = var_1384)[name = tensor("op_1385")]; - tensor var_1386 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_1386")]; - tensor filled_window_19 = add(x = var_1385, y = var_1386)[name = tensor("filled_window_19")]; - tensor var_1388 = greater_equal(x = filled_19, y = var_16)[name = tensor("op_1388")]; - tensor var_1389_dtype_0 = const()[name = tensor("op_1389_dtype_0"), val = tensor("fp32")]; - tensor var_1390 = const()[name = tensor("op_1390"), val = tensor([1, 1, 1])]; - tensor var_1389 = cast(dtype = var_1389_dtype_0, x = var_1388)[name = tensor("cast_140")]; - tensor use_roll_19 = reshape(shape = var_1390, x = var_1389)[name = tensor("use_roll_19")]; - tensor var_1392 = sub(x = var_17, y = use_roll_19)[name = tensor("op_1392")]; - tensor var_1393 = mul(x = filled_window_19, y = var_1392)[name = tensor("op_1393")]; - tensor var_1394 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_1394")]; - tensor input_201 = add(x = var_1393, y = var_1394)[name = tensor("input_201")]; - tensor var_1396 = const()[name = tensor("op_1396"), val = tensor(0x1p+0)]; - tensor var_1397 = add(x = filled_19, y = var_1396)[name = tensor("op_1397")]; - tensor clip_19 = clip(alpha = var_21, beta = var_16, x = var_1397)[name = tensor("clip_19")]; - tensor x_35_axes_0 = const()[name = tensor("x_35_axes_0"), val = tensor([-1])]; - tensor x_35 = layer_norm(axes = x_35_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_201)[name = tensor("x_35")]; - tensor input_203_perm_0 = const()[name = tensor("input_203_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_69_pad_type_0 = const()[name = tensor("inputs_69_pad_type_0"), val = tensor("valid")]; - tensor inputs_69_strides_0 = const()[name = tensor("inputs_69_strides_0"), val = tensor([1])]; - tensor inputs_69_pad_0 = const()[name = tensor("inputs_69_pad_0"), val = tensor([0, 0])]; - tensor inputs_69_dilations_0 = const()[name = tensor("inputs_69_dilations_0"), val = tensor([1])]; - tensor inputs_69_groups_0 = const()[name = tensor("inputs_69_groups_0"), val = tensor(1)]; - tensor input_203 = transpose(perm = input_203_perm_0, x = x_35)[name = tensor("transpose_36")]; - tensor inputs_69 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_69_dilations_0, groups = inputs_69_groups_0, pad = inputs_69_pad_0, pad_type = inputs_69_pad_type_0, strides = inputs_69_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_203)[name = tensor("inputs_69")]; - tensor var_1418_split_sizes_0 = const()[name = tensor("op_1418_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1418_axis_0 = const()[name = tensor("op_1418_axis_0"), val = tensor(1)]; - tensor var_1418_0, tensor var_1418_1 = split(axis = var_1418_axis_0, split_sizes = var_1418_split_sizes_0, x = inputs_69)[name = tensor("op_1418")]; - tensor var_1420 = sigmoid(x = var_1418_1)[name = tensor("op_1420")]; - tensor inputs_71 = mul(x = var_1418_0, y = var_1420)[name = tensor("inputs_71")]; - tensor outputs_aug_19_pad_type_0 = const()[name = tensor("outputs_aug_19_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_19_pad_0 = const()[name = tensor("outputs_aug_19_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_19_groups_0 = const()[name = tensor("outputs_aug_19_groups_0"), val = tensor(256)]; - tensor outputs_aug_19_strides_0 = const()[name = tensor("outputs_aug_19_strides_0"), val = tensor([1])]; - tensor outputs_aug_19_dilations_0 = const()[name = tensor("outputs_aug_19_dilations_0"), val = tensor([1])]; - tensor outputs_aug_19 = conv(dilations = outputs_aug_19_dilations_0, groups = outputs_aug_19_groups_0, pad = outputs_aug_19_pad_0, pad_type = outputs_aug_19_pad_type_0, strides = outputs_aug_19_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_71)[name = tensor("outputs_aug_19")]; - tensor input_205_begin_0 = const()[name = tensor("input_205_begin_0"), val = tensor([0, 0, 0])]; - tensor input_205_end_0 = const()[name = tensor("input_205_end_0"), val = tensor([1, 256, 16])]; - tensor input_205_end_mask_0 = const()[name = tensor("input_205_end_mask_0"), val = tensor([true, true, false])]; - tensor input_205 = slice_by_index(begin = input_205_begin_0, end = input_205_end_0, end_mask = input_205_end_mask_0, x = outputs_aug_19)[name = tensor("input_205")]; - tensor inputs_73 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_205)[name = tensor("inputs_73")]; - tensor input_207 = silu(x = inputs_73)[name = tensor("input_207")]; - tensor input_209_pad_type_0 = const()[name = tensor("input_209_pad_type_0"), val = tensor("valid")]; - tensor input_209_strides_0 = const()[name = tensor("input_209_strides_0"), val = tensor([1])]; - tensor input_209_pad_0 = const()[name = tensor("input_209_pad_0"), val = tensor([0, 0])]; - tensor input_209_dilations_0 = const()[name = tensor("input_209_dilations_0"), val = tensor([1])]; - tensor input_209_groups_0 = const()[name = tensor("input_209_groups_0"), val = tensor(1)]; - tensor input_209 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_209_dilations_0, groups = input_209_groups_0, pad = input_209_pad_0, pad_type = input_209_pad_type_0, strides = input_209_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_207)[name = tensor("input_209")]; - tensor conv_out_19_perm_0 = const()[name = tensor("conv_out_19_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1451_begin_0 = const()[name = tensor("op_1451_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1451_end_0 = const()[name = tensor("op_1451_end_0"), val = tensor([1, 16, 256])]; - tensor var_1451_end_mask_0 = const()[name = tensor("op_1451_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_19 = transpose(perm = conv_out_19_perm_0, x = input_209)[name = tensor("transpose_35")]; - tensor var_1451 = slice_by_index(begin = var_1451_begin_0, end = var_1451_end_0, end_mask = var_1451_end_mask_0, x = conv_out_19)[name = tensor("op_1451")]; - tensor var_1454_begin_0 = const()[name = tensor("op_1454_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1454_end_0 = const()[name = tensor("op_1454_end_0"), val = tensor([1, 2, 256])]; - tensor var_1454_end_mask_0 = const()[name = tensor("op_1454_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1454 = slice_by_index(begin = var_1454_begin_0, end = var_1454_end_0, end_mask = var_1454_end_mask_0, x = x_33)[name = tensor("op_1454")]; - tensor var_1457_begin_0 = const()[name = tensor("op_1457_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1457_end_0 = const()[name = tensor("op_1457_end_0"), val = tensor([1, 16, 256])]; - tensor var_1457_end_mask_0 = const()[name = tensor("op_1457_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1457 = slice_by_index(begin = var_1457_begin_0, end = var_1457_end_0, end_mask = var_1457_end_mask_0, x = input_201)[name = tensor("op_1457")]; + tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_1049)[name = tensor("x_expanded_19")]; + tensor var_1063 = sub(x = var_17, y = fill_mask_19)[name = tensor("op_1063")]; + tensor var_1064 = mul(x = window_25, y = var_1063)[name = tensor("op_1064")]; + tensor var_1065 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_1065")]; + tensor filled_window_19 = add(x = var_1064, y = var_1065)[name = tensor("filled_window_19")]; + tensor var_1067 = greater_equal(x = filled_19, y = var_16)[name = tensor("op_1067")]; + tensor var_1068_dtype_0 = const()[name = tensor("op_1068_dtype_0"), val = tensor("fp32")]; + tensor var_1069 = const()[name = tensor("op_1069"), val = tensor([1, 1, 1])]; + tensor var_1068 = cast(dtype = var_1068_dtype_0, x = var_1067)[name = tensor("cast_140")]; + tensor use_roll_19 = reshape(shape = var_1069, x = var_1068)[name = tensor("use_roll_19")]; + tensor var_1071 = sub(x = var_17, y = use_roll_19)[name = tensor("op_1071")]; + tensor var_1072 = mul(x = filled_window_19, y = var_1071)[name = tensor("op_1072")]; + tensor var_1073 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_1073")]; + tensor window_27 = add(x = var_1072, y = var_1073)[name = tensor("window_27")]; + tensor var_1075 = const()[name = tensor("op_1075"), val = tensor(0x1p+0)]; + tensor var_1076 = add(x = filled_19, y = var_1075)[name = tensor("op_1076")]; + tensor clip_19 = clip(alpha = var_21, beta = var_16, x = var_1076)[name = tensor("clip_19")]; + tensor var_1079_begin_0 = const()[name = tensor("op_1079_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1079_end_0 = const()[name = tensor("op_1079_end_0"), val = tensor([1, 2, 256])]; + tensor var_1079_end_mask_0 = const()[name = tensor("op_1079_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1079 = slice_by_index(begin = var_1079_begin_0, end = var_1079_end_0, end_mask = var_1079_end_mask_0, x = x_21)[name = tensor("op_1079")]; + tensor var_1082_begin_0 = const()[name = tensor("op_1082_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1082_end_0 = const()[name = tensor("op_1082_end_0"), val = tensor([1, 16, 256])]; + tensor var_1082_end_mask_0 = const()[name = tensor("op_1082_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1082 = slice_by_index(begin = var_1082_begin_0, end = var_1082_end_0, end_mask = var_1082_end_mask_0, x = window_27)[name = tensor("op_1082")]; tensor rolled_21_interleave_0 = const()[name = tensor("rolled_21_interleave_0"), val = tensor(false)]; - tensor rolled_21 = concat(axis = var_33, interleave = rolled_21_interleave_0, values = (var_1457, var_1454))[name = tensor("rolled_21")]; + tensor rolled_21 = concat(axis = var_33, interleave = rolled_21_interleave_0, values = (var_1082, var_1079))[name = tensor("rolled_21")]; tensor clip_20 = clip(alpha = var_21, beta = var_20, x = clip_19)[name = tensor("clip_20")]; - tensor var_1462 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_1462")]; - tensor var_1463_dtype_0 = const()[name = tensor("op_1463_dtype_0"), val = tensor("fp32")]; - tensor var_1464 = const()[name = tensor("op_1464"), val = tensor([1, 16, 1])]; - tensor var_1463 = cast(dtype = var_1463_dtype_0, x = var_1462)[name = tensor("cast_139")]; - tensor fill_mask_21 = reshape(shape = var_1464, x = var_1463)[name = tensor("fill_mask_21")]; + tensor var_1087 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_1087")]; + tensor var_1088_dtype_0 = const()[name = tensor("op_1088_dtype_0"), val = tensor("fp32")]; + tensor var_1089 = const()[name = tensor("op_1089"), val = tensor([1, 16, 1])]; + tensor var_1088 = cast(dtype = var_1088_dtype_0, x = var_1087)[name = tensor("cast_139")]; + tensor fill_mask_21 = reshape(shape = var_1089, x = var_1088)[name = tensor("fill_mask_21")]; tensor x_expanded_21_reps_0 = const()[name = tensor("x_expanded_21_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_1454)[name = tensor("x_expanded_21")]; - tensor var_1468 = sub(x = var_17, y = fill_mask_21)[name = tensor("op_1468")]; - tensor var_1469 = mul(x = input_201, y = var_1468)[name = tensor("op_1469")]; - tensor var_1470 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_1470")]; - tensor filled_window_21 = add(x = var_1469, y = var_1470)[name = tensor("filled_window_21")]; - tensor var_1472 = greater_equal(x = clip_19, y = var_16)[name = tensor("op_1472")]; - tensor var_1473_dtype_0 = const()[name = tensor("op_1473_dtype_0"), val = tensor("fp32")]; - tensor var_1474 = const()[name = tensor("op_1474"), val = tensor([1, 1, 1])]; - tensor var_1473 = cast(dtype = var_1473_dtype_0, x = var_1472)[name = tensor("cast_138")]; - tensor use_roll_21 = reshape(shape = var_1474, x = var_1473)[name = tensor("use_roll_21")]; - tensor var_1476 = sub(x = var_17, y = use_roll_21)[name = tensor("op_1476")]; - tensor var_1477 = mul(x = filled_window_21, y = var_1476)[name = tensor("op_1477")]; - tensor var_1478 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_1478")]; - tensor input_211 = add(x = var_1477, y = var_1478)[name = tensor("input_211")]; - tensor var_1480 = const()[name = tensor("op_1480"), val = tensor(0x1p+0)]; - tensor var_1481 = add(x = clip_19, y = var_1480)[name = tensor("op_1481")]; - tensor clip_21 = clip(alpha = var_21, beta = var_16, x = var_1481)[name = tensor("clip_21")]; - tensor x_37_axes_0 = const()[name = tensor("x_37_axes_0"), val = tensor([-1])]; - tensor x_37 = layer_norm(axes = x_37_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_211)[name = tensor("x_37")]; - tensor input_213_perm_0 = const()[name = tensor("input_213_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_75_pad_type_0 = const()[name = tensor("inputs_75_pad_type_0"), val = tensor("valid")]; - tensor inputs_75_strides_0 = const()[name = tensor("inputs_75_strides_0"), val = tensor([1])]; - tensor inputs_75_pad_0 = const()[name = tensor("inputs_75_pad_0"), val = tensor([0, 0])]; - tensor inputs_75_dilations_0 = const()[name = tensor("inputs_75_dilations_0"), val = tensor([1])]; - tensor inputs_75_groups_0 = const()[name = tensor("inputs_75_groups_0"), val = tensor(1)]; - tensor input_213 = transpose(perm = input_213_perm_0, x = x_37)[name = tensor("transpose_34")]; - tensor inputs_75 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_75_dilations_0, groups = inputs_75_groups_0, pad = inputs_75_pad_0, pad_type = inputs_75_pad_type_0, strides = inputs_75_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_213)[name = tensor("inputs_75")]; - tensor var_1502_split_sizes_0 = const()[name = tensor("op_1502_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1502_axis_0 = const()[name = tensor("op_1502_axis_0"), val = tensor(1)]; - tensor var_1502_0, tensor var_1502_1 = split(axis = var_1502_axis_0, split_sizes = var_1502_split_sizes_0, x = inputs_75)[name = tensor("op_1502")]; - tensor var_1504 = sigmoid(x = var_1502_1)[name = tensor("op_1504")]; - tensor inputs_77 = mul(x = var_1502_0, y = var_1504)[name = tensor("inputs_77")]; - tensor outputs_aug_21_pad_type_0 = const()[name = tensor("outputs_aug_21_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_21_pad_0 = const()[name = tensor("outputs_aug_21_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_21_groups_0 = const()[name = tensor("outputs_aug_21_groups_0"), val = tensor(256)]; - tensor outputs_aug_21_strides_0 = const()[name = tensor("outputs_aug_21_strides_0"), val = tensor([1])]; - tensor outputs_aug_21_dilations_0 = const()[name = tensor("outputs_aug_21_dilations_0"), val = tensor([1])]; - tensor outputs_aug_21 = conv(dilations = outputs_aug_21_dilations_0, groups = outputs_aug_21_groups_0, pad = outputs_aug_21_pad_0, pad_type = outputs_aug_21_pad_type_0, strides = outputs_aug_21_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_77)[name = tensor("outputs_aug_21")]; - tensor input_215_begin_0 = const()[name = tensor("input_215_begin_0"), val = tensor([0, 0, 0])]; - tensor input_215_end_0 = const()[name = tensor("input_215_end_0"), val = tensor([1, 256, 16])]; - tensor input_215_end_mask_0 = const()[name = tensor("input_215_end_mask_0"), val = tensor([true, true, false])]; - tensor input_215 = slice_by_index(begin = input_215_begin_0, end = input_215_end_0, end_mask = input_215_end_mask_0, x = outputs_aug_21)[name = tensor("input_215")]; - tensor inputs_79 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_215)[name = tensor("inputs_79")]; - tensor input_217 = silu(x = inputs_79)[name = tensor("input_217")]; - tensor input_219_pad_type_0 = const()[name = tensor("input_219_pad_type_0"), val = tensor("valid")]; - tensor input_219_strides_0 = const()[name = tensor("input_219_strides_0"), val = tensor([1])]; - tensor input_219_pad_0 = const()[name = tensor("input_219_pad_0"), val = tensor([0, 0])]; - tensor input_219_dilations_0 = const()[name = tensor("input_219_dilations_0"), val = tensor([1])]; - tensor input_219_groups_0 = const()[name = tensor("input_219_groups_0"), val = tensor(1)]; - tensor input_219 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_219_dilations_0, groups = input_219_groups_0, pad = input_219_pad_0, pad_type = input_219_pad_type_0, strides = input_219_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_217)[name = tensor("input_219")]; - tensor conv_out_21_perm_0 = const()[name = tensor("conv_out_21_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1535_begin_0 = const()[name = tensor("op_1535_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1535_end_0 = const()[name = tensor("op_1535_end_0"), val = tensor([1, 16, 256])]; - tensor var_1535_end_mask_0 = const()[name = tensor("op_1535_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_21 = transpose(perm = conv_out_21_perm_0, x = input_219)[name = tensor("transpose_33")]; - tensor var_1535 = slice_by_index(begin = var_1535_begin_0, end = var_1535_end_0, end_mask = var_1535_end_mask_0, x = conv_out_21)[name = tensor("op_1535")]; - tensor var_1538_begin_0 = const()[name = tensor("op_1538_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1538_end_0 = const()[name = tensor("op_1538_end_0"), val = tensor([1, 1, 256])]; - tensor var_1538_end_mask_0 = const()[name = tensor("op_1538_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1538 = slice_by_index(begin = var_1538_begin_0, end = var_1538_end_0, end_mask = var_1538_end_mask_0, x = x_33)[name = tensor("op_1538")]; - tensor var_1541_begin_0 = const()[name = tensor("op_1541_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1541_end_0 = const()[name = tensor("op_1541_end_0"), val = tensor([1, 16, 256])]; - tensor var_1541_end_mask_0 = const()[name = tensor("op_1541_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1541 = slice_by_index(begin = var_1541_begin_0, end = var_1541_end_0, end_mask = var_1541_end_mask_0, x = input_211)[name = tensor("op_1541")]; + tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_1079)[name = tensor("x_expanded_21")]; + tensor var_1093 = sub(x = var_17, y = fill_mask_21)[name = tensor("op_1093")]; + tensor var_1094 = mul(x = window_27, y = var_1093)[name = tensor("op_1094")]; + tensor var_1095 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_1095")]; + tensor filled_window_21 = add(x = var_1094, y = var_1095)[name = tensor("filled_window_21")]; + tensor var_1097 = greater_equal(x = clip_19, y = var_16)[name = tensor("op_1097")]; + tensor var_1098_dtype_0 = const()[name = tensor("op_1098_dtype_0"), val = tensor("fp32")]; + tensor var_1099 = const()[name = tensor("op_1099"), val = tensor([1, 1, 1])]; + tensor var_1098 = cast(dtype = var_1098_dtype_0, x = var_1097)[name = tensor("cast_138")]; + tensor use_roll_21 = reshape(shape = var_1099, x = var_1098)[name = tensor("use_roll_21")]; + tensor var_1101 = sub(x = var_17, y = use_roll_21)[name = tensor("op_1101")]; + tensor var_1102 = mul(x = filled_window_21, y = var_1101)[name = tensor("op_1102")]; + tensor var_1103 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_1103")]; + tensor window_29 = add(x = var_1102, y = var_1103)[name = tensor("window_29")]; + tensor var_1105 = const()[name = tensor("op_1105"), val = tensor(0x1p+0)]; + tensor var_1106 = add(x = clip_19, y = var_1105)[name = tensor("op_1106")]; + tensor clip_21 = clip(alpha = var_21, beta = var_16, x = var_1106)[name = tensor("clip_21")]; + tensor var_1109_begin_0 = const()[name = tensor("op_1109_begin_0"), val = tensor([0, 2, 0])]; + tensor var_1109_end_0 = const()[name = tensor("op_1109_end_0"), val = tensor([1, 1, 256])]; + tensor var_1109_end_mask_0 = const()[name = tensor("op_1109_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1109 = slice_by_index(begin = var_1109_begin_0, end = var_1109_end_0, end_mask = var_1109_end_mask_0, x = x_21)[name = tensor("op_1109")]; + tensor var_1112_begin_0 = const()[name = tensor("op_1112_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1112_end_0 = const()[name = tensor("op_1112_end_0"), val = tensor([1, 16, 256])]; + tensor var_1112_end_mask_0 = const()[name = tensor("op_1112_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1112 = slice_by_index(begin = var_1112_begin_0, end = var_1112_end_0, end_mask = var_1112_end_mask_0, x = window_29)[name = tensor("op_1112")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1541, var_1538))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1112, var_1109))[name = tensor("rolled")]; tensor clip_22 = clip(alpha = var_21, beta = var_20, x = clip_21)[name = tensor("clip_22")]; - tensor var_1546 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_1546")]; - tensor var_1547_dtype_0 = const()[name = tensor("op_1547_dtype_0"), val = tensor("fp32")]; - tensor var_1548 = const()[name = tensor("op_1548"), val = tensor([1, 16, 1])]; - tensor var_1547 = cast(dtype = var_1547_dtype_0, x = var_1546)[name = tensor("cast_137")]; - tensor fill_mask = reshape(shape = var_1548, x = var_1547)[name = tensor("fill_mask")]; + tensor var_1117 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_1117")]; + tensor var_1118_dtype_0 = const()[name = tensor("op_1118_dtype_0"), val = tensor("fp32")]; + tensor var_1119 = const()[name = tensor("op_1119"), val = tensor([1, 16, 1])]; + tensor var_1118 = cast(dtype = var_1118_dtype_0, x = var_1117)[name = tensor("cast_137")]; + tensor fill_mask = reshape(shape = var_1119, x = var_1118)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1538)[name = tensor("x_expanded")]; - tensor var_1552 = sub(x = var_17, y = fill_mask)[name = tensor("op_1552")]; - tensor var_1553 = mul(x = input_211, y = var_1552)[name = tensor("op_1553")]; - tensor var_1554 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1554")]; - tensor filled_window = add(x = var_1553, y = var_1554)[name = tensor("filled_window")]; - tensor var_1556 = greater_equal(x = clip_21, y = var_16)[name = tensor("op_1556")]; - tensor var_1557_dtype_0 = const()[name = tensor("op_1557_dtype_0"), val = tensor("fp32")]; - tensor var_1558 = const()[name = tensor("op_1558"), val = tensor([1, 1, 1])]; - tensor var_1557 = cast(dtype = var_1557_dtype_0, x = var_1556)[name = tensor("cast_136")]; - tensor use_roll = reshape(shape = var_1558, x = var_1557)[name = tensor("use_roll")]; - tensor var_1560 = sub(x = var_17, y = use_roll)[name = tensor("op_1560")]; - tensor var_1561 = mul(x = filled_window, y = var_1560)[name = tensor("op_1561")]; - tensor var_1562 = mul(x = rolled, y = use_roll)[name = tensor("op_1562")]; - tensor input_221 = add(x = var_1561, y = var_1562)[name = tensor("input_221")]; - tensor var_1564 = const()[name = tensor("op_1564"), val = tensor(0x1p+0)]; - tensor var_1565 = add(x = clip_21, y = var_1564)[name = tensor("op_1565")]; - tensor clip_23 = clip(alpha = var_21, beta = var_16, x = var_1565)[name = tensor("clip_23")]; - tensor x_39_axes_0 = const()[name = tensor("x_39_axes_0"), val = tensor([-1])]; - tensor x_39 = layer_norm(axes = x_39_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_221)[name = tensor("x_39")]; - tensor input_223_perm_0 = const()[name = tensor("input_223_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_81_pad_type_0 = const()[name = tensor("inputs_81_pad_type_0"), val = tensor("valid")]; - tensor inputs_81_strides_0 = const()[name = tensor("inputs_81_strides_0"), val = tensor([1])]; - tensor inputs_81_pad_0 = const()[name = tensor("inputs_81_pad_0"), val = tensor([0, 0])]; - tensor inputs_81_dilations_0 = const()[name = tensor("inputs_81_dilations_0"), val = tensor([1])]; - tensor inputs_81_groups_0 = const()[name = tensor("inputs_81_groups_0"), val = tensor(1)]; - tensor input_223 = transpose(perm = input_223_perm_0, x = x_39)[name = tensor("transpose_32")]; - tensor inputs_81 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_81_dilations_0, groups = inputs_81_groups_0, pad = inputs_81_pad_0, pad_type = inputs_81_pad_type_0, strides = inputs_81_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_223)[name = tensor("inputs_81")]; - tensor var_1586_split_sizes_0 = const()[name = tensor("op_1586_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1586_axis_0 = const()[name = tensor("op_1586_axis_0"), val = tensor(1)]; - tensor var_1586_0, tensor var_1586_1 = split(axis = var_1586_axis_0, split_sizes = var_1586_split_sizes_0, x = inputs_81)[name = tensor("op_1586")]; - tensor var_1588 = sigmoid(x = var_1586_1)[name = tensor("op_1588")]; - tensor inputs_83 = mul(x = var_1586_0, y = var_1588)[name = tensor("inputs_83")]; + tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1109)[name = tensor("x_expanded")]; + tensor var_1123 = sub(x = var_17, y = fill_mask)[name = tensor("op_1123")]; + tensor var_1124 = mul(x = window_29, y = var_1123)[name = tensor("op_1124")]; + tensor var_1125 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1125")]; + tensor filled_window = add(x = var_1124, y = var_1125)[name = tensor("filled_window")]; + tensor var_1127 = greater_equal(x = clip_21, y = var_16)[name = tensor("op_1127")]; + tensor var_1128_dtype_0 = const()[name = tensor("op_1128_dtype_0"), val = tensor("fp32")]; + tensor var_1129 = const()[name = tensor("op_1129"), val = tensor([1, 1, 1])]; + tensor var_1128 = cast(dtype = var_1128_dtype_0, x = var_1127)[name = tensor("cast_136")]; + tensor use_roll = reshape(shape = var_1129, x = var_1128)[name = tensor("use_roll")]; + tensor var_1131 = sub(x = var_17, y = use_roll)[name = tensor("op_1131")]; + tensor var_1132 = mul(x = filled_window, y = var_1131)[name = tensor("op_1132")]; + tensor var_1133 = mul(x = rolled, y = use_roll)[name = tensor("op_1133")]; + tensor window = add(x = var_1132, y = var_1133)[name = tensor("window")]; + tensor var_1135 = const()[name = tensor("op_1135"), val = tensor(0x1p+0)]; + tensor var_1136 = add(x = clip_21, y = var_1135)[name = tensor("op_1136")]; + tensor clip_23 = clip(alpha = var_21, beta = var_16, x = var_1136)[name = tensor("clip_23")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_30, interleave = input_141_interleave_0, values = (window_27, window_29, window))[name = tensor("input_141")]; + tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; + tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; + tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_33_pad_type_0 = const()[name = tensor("inputs_33_pad_type_0"), val = tensor("valid")]; + tensor inputs_33_strides_0 = const()[name = tensor("inputs_33_strides_0"), val = tensor([1])]; + tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; + tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; + tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; + tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; + tensor var_1159_split_sizes_0 = const()[name = tensor("op_1159_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1159_axis_0 = const()[name = tensor("op_1159_axis_0"), val = tensor(1)]; + tensor var_1159_0, tensor var_1159_1 = split(axis = var_1159_axis_0, split_sizes = var_1159_split_sizes_0, x = inputs_33)[name = tensor("op_1159")]; + tensor var_1161 = sigmoid(x = var_1159_1)[name = tensor("op_1161")]; + tensor inputs_35 = mul(x = var_1159_0, y = var_1161)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; tensor outputs_aug_strides_0 = const()[name = tensor("outputs_aug_strides_0"), val = tensor([1])]; tensor outputs_aug_dilations_0 = const()[name = tensor("outputs_aug_dilations_0"), val = tensor([1])]; - tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_83)[name = tensor("outputs_aug")]; - tensor input_225_begin_0 = const()[name = tensor("input_225_begin_0"), val = tensor([0, 0, 0])]; - tensor input_225_end_0 = const()[name = tensor("input_225_end_0"), val = tensor([1, 256, 16])]; - tensor input_225_end_mask_0 = const()[name = tensor("input_225_end_mask_0"), val = tensor([true, true, false])]; - tensor input_225 = slice_by_index(begin = input_225_begin_0, end = input_225_end_0, end_mask = input_225_end_mask_0, x = outputs_aug)[name = tensor("input_225")]; - tensor inputs_85 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_225)[name = tensor("inputs_85")]; - tensor input_227 = silu(x = inputs_85)[name = tensor("input_227")]; - tensor input_229_pad_type_0 = const()[name = tensor("input_229_pad_type_0"), val = tensor("valid")]; - tensor input_229_strides_0 = const()[name = tensor("input_229_strides_0"), val = tensor([1])]; - tensor input_229_pad_0 = const()[name = tensor("input_229_pad_0"), val = tensor([0, 0])]; - tensor input_229_dilations_0 = const()[name = tensor("input_229_dilations_0"), val = tensor([1])]; - tensor input_229_groups_0 = const()[name = tensor("input_229_groups_0"), val = tensor(1)]; - tensor input_229 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_229_dilations_0, groups = input_229_groups_0, pad = input_229_pad_0, pad_type = input_229_pad_type_0, strides = input_229_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_227)[name = tensor("input_229")]; - tensor conv_out_23_perm_0 = const()[name = tensor("conv_out_23_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1619_begin_0 = const()[name = tensor("op_1619_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1619_end_0 = const()[name = tensor("op_1619_end_0"), val = tensor([1, 16, 256])]; - tensor var_1619_end_mask_0 = const()[name = tensor("op_1619_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_23 = transpose(perm = conv_out_23_perm_0, x = input_229)[name = tensor("transpose_31")]; - tensor var_1619 = slice_by_index(begin = var_1619_begin_0, end = var_1619_end_0, end_mask = var_1619_end_mask_0, x = conv_out_23)[name = tensor("op_1619")]; - tensor var_1622_interleave_0 = const()[name = tensor("op_1622_interleave_0"), val = tensor(false)]; - tensor var_1622 = concat(axis = var_33, interleave = var_1622_interleave_0, values = (var_1451, var_1535, var_1619))[name = tensor("op_1622")]; - tensor input_231 = add(x = x_33, y = var_1622)[name = tensor("input_231")]; - tensor input_233_axes_0 = const()[name = tensor("input_233_axes_0"), val = tensor([-1])]; - tensor input_233 = layer_norm(axes = input_233_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_231)[name = tensor("input_233")]; - tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_233)[name = tensor("linear_35")]; - tensor input_235 = silu(x = inputs)[name = tensor("input_235")]; - tensor input_239 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_235)[name = tensor("linear_36")]; - tensor var_1645 = const()[name = tensor("op_1645"), val = tensor(0x1p-1)]; - tensor var_1646 = mul(x = input_239, y = var_1645)[name = tensor("op_1646")]; - tensor input_241 = add(x = var_1646, y = input_231)[name = tensor("input_241")]; - tensor x_41_axes_0 = const()[name = tensor("x_41_axes_0"), val = tensor([-1])]; - tensor x_41 = layer_norm(axes = x_41_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_241)[name = tensor("x_41")]; + tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_35)[name = tensor("outputs_aug")]; + tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; + tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([3, 256, 16])]; + tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; + tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug)[name = tensor("input_145")]; + tensor inputs_37 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_145)[name = tensor("inputs_37")]; + tensor input_147 = silu(x = inputs_37)[name = tensor("input_147")]; + tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; + tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; + tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; + tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; + tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; + tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; + tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1192_begin_0 = const()[name = tensor("op_1192_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1192_end_0 = const()[name = tensor("op_1192_end_0"), val = tensor([3, 16, 256])]; + tensor var_1192_end_mask_0 = const()[name = tensor("op_1192_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_1192 = slice_by_index(begin = var_1192_begin_0, end = var_1192_end_0, end_mask = var_1192_end_mask_0, x = conv_out_7)[name = tensor("op_1192")]; + tensor var_1194_perm_0 = const()[name = tensor("op_1194_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1194 = transpose(perm = var_1194_perm_0, x = var_1192)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_1194)[name = tensor("input_151")]; + tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; + tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; + tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; + tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; + tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; + tensor var_1217 = const()[name = tensor("op_1217"), val = tensor(0x1p-1)]; + tensor var_1218 = mul(x = input_159, y = var_1217)[name = tensor("op_1218")]; + tensor input_161 = add(x = var_1218, y = input_151)[name = tensor("input_161")]; + tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; + tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; tensor cat_interleave_0 = const()[name = tensor("cat_interleave_0"), val = tensor(false)]; - tensor x_ct = transpose(perm = x_ct_perm_0, x = x_41)[name = tensor("transpose_30")]; + tensor x_ct = transpose(perm = x_ct_perm_0, x = x_25)[name = tensor("transpose_30")]; tensor cat = concat(axis = var_27, interleave = cat_interleave_0, values = (cnn_window, x_ct))[name = tensor("cat")]; tensor conv_out_pad_type_0 = const()[name = tensor("conv_out_pad_type_0"), val = tensor("valid")]; tensor conv_out_strides_0 = const()[name = tensor("conv_out_strides_0"), val = tensor([1])]; @@ -1513,44 +1210,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_1664_begin_0 = const()[name = tensor("op_1664_begin_0"), val = tensor([0, 0, 3])]; - tensor var_1664_end_0 = const()[name = tensor("op_1664_end_0"), val = tensor([1, 256, 21])]; - tensor var_1664_end_mask_0 = const()[name = tensor("op_1664_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_1664_begin_0, end = var_1664_end_0, end_mask = var_1664_end_mask_0, x = cat)[name = tensor("op_1664")]; - tensor input_243_perm_0 = const()[name = tensor("input_243_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1666 = const()[name = tensor("op_1666"), val = tensor([-1])]; - tensor input_243 = transpose(perm = input_243_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_1667 = reduce_l2_norm(axes = var_1666, keep_dims = var_36, x = input_243)[name = tensor("op_1667")]; - tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; - tensor clip_24 = clip(alpha = var_12, beta = const_20, x = var_1667)[name = tensor("clip_24")]; - tensor emb = real_div(x = input_243, y = clip_24)[name = tensor("emb")]; - tensor var_1671_axis_0 = const()[name = tensor("op_1671_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_1671_axis_0, values = (var_214, var_594, var_974, nkv_1))[name = tensor("op_1671")]; - tensor var_1673_axis_0 = const()[name = tensor("op_1673_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_1673_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1673")]; - tensor var_1675_axis_0 = const()[name = tensor("op_1675_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_1675_axis_0, values = (input_41, input_101, input_161, input_221))[name = tensor("op_1675")]; - tensor var_1677_axis_0 = const()[name = tensor("op_1677_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_1677_axis_0, values = (clip_5, clip_11, clip_17, clip_23))[name = tensor("op_1677")]; - tensor var_1687 = const()[name = tensor("op_1687"), val = tensor(0x1.5798eep-27)]; - tensor var_1693 = const()[name = tensor("op_1693"), val = tensor(0x1.4f8b58p-17)]; - tensor var_1695 = const()[name = tensor("op_1695"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_1696 = const()[name = tensor("op_1696"), val = tensor(true)]; - tensor var_1698 = const()[name = tensor("op_1698"), val = tensor(0x1p+0)]; - tensor var_1702 = const()[name = tensor("op_1702"), val = tensor(-1)]; - tensor var_1707 = const()[name = tensor("op_1707"), val = tensor(0)]; + tensor var_1236_begin_0 = const()[name = tensor("op_1236_begin_0"), val = tensor([0, 0, 3])]; + tensor var_1236_end_0 = const()[name = tensor("op_1236_end_0"), val = tensor([1, 256, 21])]; + tensor var_1236_end_mask_0 = const()[name = tensor("op_1236_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_1236_begin_0, end = var_1236_end_0, end_mask = var_1236_end_mask_0, x = cat)[name = tensor("op_1236")]; + tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1238 = const()[name = tensor("op_1238"), val = tensor([-1])]; + tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; + tensor var_1239 = reduce_l2_norm(axes = var_1238, keep_dims = var_36, x = input_163)[name = tensor("op_1239")]; + tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; + tensor clip_24 = clip(alpha = var_12, beta = const_12, x = var_1239)[name = tensor("clip_24")]; + tensor emb = real_div(x = input_163, y = clip_24)[name = tensor("emb")]; + tensor var_1243_axis_0 = const()[name = tensor("op_1243_axis_0"), val = tensor(0)]; + tensor enc_kv_new = stack(axis = var_1243_axis_0, values = (var_214, var_487, var_760, nkv_1))[name = tensor("op_1243")]; + tensor var_1245_axis_0 = const()[name = tensor("op_1245_axis_0"), val = tensor(0)]; + tensor enc_scale_new = stack(axis = var_1245_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1245")]; + tensor var_1247_axis_0 = const()[name = tensor("op_1247_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1247_axis_0, values = (window_7, window_15, window_23, window))[name = tensor("op_1247")]; + tensor var_1249_axis_0 = const()[name = tensor("op_1249_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1249_axis_0, values = (clip_5, clip_11, clip_17, clip_23))[name = tensor("op_1249")]; + tensor var_1259 = const()[name = tensor("op_1259"), val = tensor(0x1.5798eep-27)]; + tensor var_1264 = const()[name = tensor("op_1264"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1266 = const()[name = tensor("op_1266"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1267 = const()[name = tensor("op_1267"), val = tensor(true)]; + tensor var_1269 = const()[name = tensor("op_1269"), val = tensor(0x1p+0)]; + tensor var_1273 = const()[name = tensor("op_1273"), val = tensor(-1)]; + tensor var_1279 = const()[name = tensor("op_1279"), val = tensor(0)]; tensor pos = const()[name = tensor("pos"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395456)))]; - tensor var_1769_axes_0 = const()[name = tensor("op_1769_axes_0"), val = tensor([2])]; - tensor var_1769 = expand_dims(axes = var_1769_axes_0, x = emb)[name = tensor("op_1769")]; - tensor var_1770 = const()[name = tensor("op_1770"), val = tensor([1, 1, 12, 1])]; - tensor emb_exp = tile(reps = var_1770, x = var_1769)[name = tensor("emb_exp")]; - tensor input_245_interleave_0 = const()[name = tensor("input_245_interleave_0"), val = tensor(false)]; - tensor input_245 = concat(axis = var_1702, interleave = input_245_interleave_0, values = (emb_exp, pos))[name = tensor("input_245")]; - tensor x_43 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_245)[name = tensor("linear_37")]; - tensor var_1777_perm_0 = const()[name = tensor("op_1777_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1781 = const()[name = tensor("op_1781"), val = tensor([12, 3, 256])]; - tensor var_1777 = transpose(perm = var_1777_perm_0, x = x_43)[name = tensor("transpose_28")]; - tensor x_45 = reshape(shape = var_1781, x = var_1777)[name = tensor("x_45")]; + tensor var_1341_axes_0 = const()[name = tensor("op_1341_axes_0"), val = tensor([2])]; + tensor var_1341 = expand_dims(axes = var_1341_axes_0, x = emb)[name = tensor("op_1341")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 12, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1341)[name = tensor("emb_exp")]; + tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; + tensor input_165 = concat(axis = var_1273, interleave = input_165_interleave_0, values = (emb_exp, pos))[name = tensor("input_165")]; + tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; + tensor var_1349_perm_0 = const()[name = tensor("op_1349_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1353 = const()[name = tensor("op_1353"), val = tensor([12, 3, 256])]; + tensor var_1349 = transpose(perm = var_1349_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1353, x = var_1349)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 12, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1561,132 +1258,132 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_1789 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_45)[name = tensor("linear_38")]; - tensor var_1790 = const()[name = tensor("op_1790"), val = tensor([12, 3, 4, 64])]; - tensor var_1791 = reshape(shape = var_1790, x = var_1789)[name = tensor("op_1791")]; + tensor var_1361 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1362 = const()[name = tensor("op_1362"), val = tensor([12, 3, 4, 64])]; + tensor var_1363 = reshape(shape = var_1362, x = var_1361)[name = tensor("op_1363")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1795 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_45)[name = tensor("linear_39")]; - tensor var_1796 = const()[name = tensor("op_1796"), val = tensor(0x1p-3)]; - tensor var_1797 = mul(x = var_1795, y = var_1796)[name = tensor("op_1797")]; - tensor var_1798 = const()[name = tensor("op_1798"), val = tensor([12, 3, 4, 64])]; - tensor var_1799 = reshape(shape = var_1798, x = var_1797)[name = tensor("op_1799")]; + tensor var_1367 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1368 = const()[name = tensor("op_1368"), val = tensor(0x1p-3)]; + tensor var_1369 = mul(x = var_1367, y = var_1368)[name = tensor("op_1369")]; + tensor var_1370 = const()[name = tensor("op_1370"), val = tensor([12, 3, 4, 64])]; + tensor var_1371 = reshape(shape = var_1370, x = var_1369)[name = tensor("op_1371")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1803 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_45)[name = tensor("linear_40")]; - tensor var_1804 = const()[name = tensor("op_1804"), val = tensor([12, 3, 4, 64])]; - tensor var_1805 = reshape(shape = var_1804, x = var_1803)[name = tensor("op_1805")]; + tensor var_1375 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1376 = const()[name = tensor("op_1376"), val = tensor([12, 3, 4, 64])]; + tensor var_1377 = reshape(shape = var_1376, x = var_1375)[name = tensor("op_1377")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_249 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_45)[name = tensor("linear_41")]; + tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_1707, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1279, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; - tensor const_28 = const()[name = tensor("const_28"), val = tensor(0x1.fffffep+127)]; - tensor clip_25 = clip(alpha = var_1698, beta = const_28, x = s_t_9)[name = tensor("clip_25")]; + tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; + tensor clip_25 = clip(alpha = var_1269, beta = const_20, x = s_t_9)[name = tensor("clip_25")]; tensor sqrt_s_t_9 = sqrt(x = clip_25)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_1799)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_1791)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1371)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1363)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_1817 = const()[name = tensor("op_1817"), val = tensor([1, 3])]; - tensor var_1818 = reshape(shape = var_1817, x = valid_mask)[name = tensor("op_1818")]; - tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1818)[name = tensor("causal_with_valid_1")]; - tensor var_1820 = const()[name = tensor("op_1820"), val = tensor([3, 1])]; - tensor var_1821 = reshape(shape = var_1820, x = sqrt_s_t_9)[name = tensor("op_1821")]; - tensor M_9 = real_div(x = causal_with_valid_1, y = var_1821)[name = tensor("M_9")]; - tensor var_1823 = mul(x = qk_9, y = M_9)[name = tensor("op_1823")]; + tensor var_1389 = const()[name = tensor("op_1389"), val = tensor([1, 3])]; + tensor var_1390 = reshape(shape = var_1389, x = valid_mask)[name = tensor("op_1390")]; + tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1390)[name = tensor("causal_with_valid_1")]; + tensor var_1392 = const()[name = tensor("op_1392"), val = tensor([3, 1])]; + tensor var_1393 = reshape(shape = var_1392, x = sqrt_s_t_9)[name = tensor("op_1393")]; + tensor M_9 = real_div(x = causal_with_valid_1, y = var_1393)[name = tensor("M_9")]; + tensor var_1395 = mul(x = qk_9, y = M_9)[name = tensor("op_1395")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_1805)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1823, y = v_9)[name = tensor("inner_9")]; - tensor var_1825_transpose_x_0 = const()[name = tensor("op_1825_transpose_x_0"), val = tensor(false)]; - tensor var_1825_transpose_y_0 = const()[name = tensor("op_1825_transpose_y_0"), val = tensor(false)]; - tensor var_1825 = matmul(transpose_x = var_1825_transpose_x_0, transpose_y = var_1825_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1825")]; - tensor var_1826 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1826")]; - tensor var_1827 = const()[name = tensor("op_1827"), val = tensor([1, 1, 3, 1])]; - tensor var_1828 = reshape(shape = var_1827, x = var_1826)[name = tensor("op_1828")]; - tensor cross_9 = mul(x = var_1825, y = var_1828)[name = tensor("cross_9")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1377)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1395, y = v_9)[name = tensor("inner_9")]; + tensor var_1397_transpose_x_0 = const()[name = tensor("op_1397_transpose_x_0"), val = tensor(false)]; + tensor var_1397_transpose_y_0 = const()[name = tensor("op_1397_transpose_y_0"), val = tensor(false)]; + tensor var_1397 = matmul(transpose_x = var_1397_transpose_x_0, transpose_y = var_1397_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1397")]; + tensor var_1398 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1398")]; + tensor var_1399 = const()[name = tensor("op_1399"), val = tensor([1, 1, 3, 1])]; + tensor var_1400 = reshape(shape = var_1399, x = var_1398)[name = tensor("op_1400")]; + tensor cross_9 = mul(x = var_1397, y = var_1400)[name = tensor("cross_9")]; tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; - tensor var_1831 = const()[name = tensor("op_1831"), val = tensor([1, 1, 3, 1])]; - tensor var_1832 = reshape(shape = var_1831, x = valid_mask)[name = tensor("op_1832")]; - tensor v_masked_1 = mul(x = v_9, y = var_1832)[name = tensor("v_masked_1")]; - tensor var_1834 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1834")]; - tensor var_1836_transpose_x_1 = const()[name = tensor("op_1836_transpose_x_1"), val = tensor(true)]; - tensor var_1836_transpose_y_1 = const()[name = tensor("op_1836_transpose_y_1"), val = tensor(false)]; - tensor var_1836 = matmul(transpose_x = var_1836_transpose_x_1, transpose_y = var_1836_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1836")]; - tensor new_kv_unnorm_9 = add(x = var_1834, y = var_1836)[name = tensor("new_kv_unnorm_9")]; - tensor var_1838_keep_dims_0 = const()[name = tensor("op_1838_keep_dims_0"), val = tensor(false)]; - tensor var_1838 = reduce_sum(keep_dims = var_1838_keep_dims_0, x = valid_mask)[name = tensor("op_1838")]; - tensor var_1839 = const()[name = tensor("op_1839"), val = tensor([1])]; - tensor var_1840 = reshape(shape = var_1839, x = var_1838)[name = tensor("op_1840")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_1840)[name = tensor("new_scale_9")]; - tensor const_29 = const()[name = tensor("const_29"), val = tensor(0x1.fffffep+127)]; - tensor clip_26 = clip(alpha = var_1698, beta = const_29, x = new_scale_9)[name = tensor("clip_26")]; + tensor var_1403 = const()[name = tensor("op_1403"), val = tensor([1, 1, 3, 1])]; + tensor var_1404 = reshape(shape = var_1403, x = valid_mask)[name = tensor("op_1404")]; + tensor v_masked_1 = mul(x = v_9, y = var_1404)[name = tensor("v_masked_1")]; + tensor var_1406 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1406")]; + tensor var_1408_transpose_x_1 = const()[name = tensor("op_1408_transpose_x_1"), val = tensor(true)]; + tensor var_1408_transpose_y_1 = const()[name = tensor("op_1408_transpose_y_1"), val = tensor(false)]; + tensor var_1408 = matmul(transpose_x = var_1408_transpose_x_1, transpose_y = var_1408_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1408")]; + tensor new_kv_unnorm_9 = add(x = var_1406, y = var_1408)[name = tensor("new_kv_unnorm_9")]; + tensor var_1410_keep_dims_0 = const()[name = tensor("op_1410_keep_dims_0"), val = tensor(false)]; + tensor var_1410 = reduce_sum(keep_dims = var_1410_keep_dims_0, x = valid_mask)[name = tensor("op_1410")]; + tensor var_1411 = const()[name = tensor("op_1411"), val = tensor([1])]; + tensor var_1412 = reshape(shape = var_1411, x = var_1410)[name = tensor("op_1412")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1412)[name = tensor("new_scale_9")]; + tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; + tensor clip_26 = clip(alpha = var_1269, beta = const_21, x = new_scale_9)[name = tensor("clip_26")]; tensor sqrt_new_scale_1 = sqrt(x = clip_26)[name = tensor("sqrt_new_scale_1")]; - tensor var_1844 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1844")]; - tensor var_1845_perm_0 = const()[name = tensor("op_1845_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1416 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1416")]; + tensor var_1417_perm_0 = const()[name = tensor("op_1417_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_1845 = transpose(perm = var_1845_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1695, x = var_1845)[name = tensor("out_27")]; - tensor var_1849 = const()[name = tensor("op_1849"), val = tensor([12, 3, 256])]; - tensor out_29 = reshape(shape = var_1849, x = out_27)[name = tensor("out_29")]; - tensor var_1851 = silu(x = input_249)[name = tensor("op_1851")]; - tensor input_251 = mul(x = var_1851, y = out_29)[name = tensor("input_251")]; - tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_251)[name = tensor("linear_42")]; - tensor input_253 = add(x = x_45, y = ret_out_9)[name = tensor("input_253")]; + tensor var_1417 = transpose(perm = var_1417_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1266, x = var_1417)[name = tensor("out_27")]; + tensor var_1421 = const()[name = tensor("op_1421"), val = tensor([12, 3, 256])]; + tensor out_29 = reshape(shape = var_1421, x = out_27)[name = tensor("out_29")]; + tensor var_1423 = silu(x = input_169)[name = tensor("op_1423")]; + tensor input_171 = mul(x = var_1423, y = out_29)[name = tensor("input_171")]; + tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; + tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1693, gamma = decoder_norm11_0_weight, x = input_253)[name = tensor("xt_1")]; - tensor var_1861 = const()[name = tensor("op_1861"), val = tensor([1, 12, 3, 256])]; - tensor var_1862 = reshape(shape = var_1861, x = xt_1)[name = tensor("op_1862")]; - tensor var_1863_perm_0 = const()[name = tensor("op_1863_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1866 = const()[name = tensor("op_1866"), val = tensor([3, 12, 256])]; - tensor var_1863 = transpose(perm = var_1863_perm_0, x = var_1862)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_1866, x = var_1863)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1264, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1433 = const()[name = tensor("op_1433"), val = tensor([1, 12, 3, 256])]; + tensor var_1434 = reshape(shape = var_1433, x = xt_1)[name = tensor("op_1434")]; + tensor var_1435_perm_0 = const()[name = tensor("op_1435_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1438 = const()[name = tensor("op_1438"), val = tensor([3, 12, 256])]; + tensor var_1435 = transpose(perm = var_1435_perm_0, x = var_1434)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1438, x = var_1435)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_1889 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_12 = const()[name = tensor("concat_12"), val = tensor([12, 3, 3, 256])]; - tensor var_1891 = reshape(shape = concat_12, x = var_1889)[name = tensor("op_1891")]; - tensor var_1892_axes_0 = const()[name = tensor("op_1892_axes_0"), val = tensor([0])]; - tensor var_1892 = expand_dims(axes = var_1892_axes_0, x = var_1891)[name = tensor("op_1892")]; - tensor var_1893_perm_0 = const()[name = tensor("op_1893_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_1894_axes_0 = const()[name = tensor("op_1894_axes_0"), val = tensor([-2])]; - tensor var_1893 = transpose(perm = var_1893_perm_0, x = var_1892)[name = tensor("transpose_21")]; - tensor var_1894 = squeeze(axes = var_1894_axes_0, x = var_1893)[name = tensor("op_1894")]; + tensor var_1461 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_13 = const()[name = tensor("concat_13"), val = tensor([12, 3, 3, 256])]; + tensor var_1463 = reshape(shape = concat_13, x = var_1461)[name = tensor("op_1463")]; + tensor var_1464_axes_0 = const()[name = tensor("op_1464_axes_0"), val = tensor([0])]; + tensor var_1464 = expand_dims(axes = var_1464_axes_0, x = var_1463)[name = tensor("op_1464")]; + tensor var_1465_perm_0 = const()[name = tensor("op_1465_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1466_axes_0 = const()[name = tensor("op_1466_axes_0"), val = tensor([-2])]; + tensor var_1465 = transpose(perm = var_1465_perm_0, x = var_1464)[name = tensor("transpose_21")]; + tensor var_1466 = squeeze(axes = var_1466_axes_0, x = var_1465)[name = tensor("op_1466")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 12, 3, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1894)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1466)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 12, 3, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1894)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1466)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 12, 3, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1894)[name = tensor("v_11")]; - tensor var_1902 = const()[name = tensor("op_1902"), val = tensor([12, 12, 64])]; - tensor var_1903 = reshape(shape = var_1902, x = q_11)[name = tensor("op_1903")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1466)[name = tensor("v_11")]; + tensor var_1474 = const()[name = tensor("op_1474"), val = tensor([12, 12, 64])]; + tensor var_1475 = reshape(shape = var_1474, x = q_11)[name = tensor("op_1475")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1909 = const()[name = tensor("op_1909"), val = tensor([12, 12, 64])]; - tensor var_1910 = reshape(shape = var_1909, x = k_11)[name = tensor("op_1910")]; + tensor var_1481 = const()[name = tensor("op_1481"), val = tensor([12, 12, 64])]; + tensor var_1482 = reshape(shape = var_1481, x = k_11)[name = tensor("op_1482")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1916 = const()[name = tensor("op_1916"), val = tensor([12, 12, 64])]; - tensor var_1917 = reshape(shape = var_1916, x = v_11)[name = tensor("op_1917")]; + tensor var_1488 = const()[name = tensor("op_1488"), val = tensor([12, 12, 64])]; + tensor var_1489 = reshape(shape = var_1488, x = v_11)[name = tensor("op_1489")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_1920 = const()[name = tensor("op_1920"), val = tensor([3, 4, 12, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_1903)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_1920, x = q_13)[name = tensor("q_15")]; - tensor var_1922 = const()[name = tensor("op_1922"), val = tensor([3, 4, 12, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_1910)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_1922, x = k_13)[name = tensor("k_15")]; - tensor var_1924 = const()[name = tensor("op_1924"), val = tensor([3, 4, 12, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_1917)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_1924, x = v_13)[name = tensor("v_15")]; + tensor var_1492 = const()[name = tensor("op_1492"), val = tensor([3, 4, 12, 64])]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1475)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1492, x = q_13)[name = tensor("q_15")]; + tensor var_1494 = const()[name = tensor("op_1494"), val = tensor([3, 4, 12, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1482)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1494, x = k_13)[name = tensor("k_15")]; + tensor var_1496 = const()[name = tensor("op_1496"), val = tensor([3, 4, 12, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1489)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1496, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -1697,30 +1394,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_1927 = const()[name = tensor("op_1927"), val = tensor([2, 0, 1, 3])]; - tensor var_1932 = const()[name = tensor("op_1932"), val = tensor([36, 256])]; - tensor var_1928 = transpose(perm = var_1927, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_1932, x = var_1928)[name = tensor("attn_output_3")]; + tensor var_1499 = const()[name = tensor("op_1499"), val = tensor([2, 0, 1, 3])]; + tensor var_1504 = const()[name = tensor("op_1504"), val = tensor([36, 256])]; + tensor var_1500 = transpose(perm = var_1499, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1504, x = var_1500)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_1936 = const()[name = tensor("op_1936"), val = tensor([12, 3, 256])]; - tensor attn_output_7 = reshape(shape = var_1936, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1508 = const()[name = tensor("op_1508"), val = tensor([12, 3, 256])]; + tensor attn_output_7 = reshape(shape = var_1508, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; - tensor input_255 = add(x = query_1, y = sa2_out_1)[name = tensor("input_255")]; - tensor input_257_axes_0 = const()[name = tensor("input_257_axes_0"), val = tensor([-1])]; - tensor input_257 = layer_norm(axes = input_257_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1693, gamma = decoder_norm21_0_weight, x = input_255)[name = tensor("input_257")]; - tensor input_259 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_257)[name = tensor("linear_45")]; - tensor input_261 = relu(x = input_259)[name = tensor("input_261")]; - tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_261)[name = tensor("linear_46")]; - tensor input_263 = add(x = input_257, y = ffn_out_1)[name = tensor("input_263")]; + tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; + tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1264, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; + tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; + tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; + tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1693, gamma = decoder_norm22_0_weight, x = input_263)[name = tensor("xt_3")]; - tensor var_1956 = const()[name = tensor("op_1956"), val = tensor([1, 3, 12, 256])]; - tensor x_47 = reshape(shape = var_1956, x = xt_3)[name = tensor("x_47")]; - tensor var_1958_perm_0 = const()[name = tensor("op_1958_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1962 = const()[name = tensor("op_1962"), val = tensor([12, 3, 256])]; - tensor var_1958 = transpose(perm = var_1958_perm_0, x = x_47)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_1962, x = var_1958)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1264, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1528 = const()[name = tensor("op_1528"), val = tensor([1, 3, 12, 256])]; + tensor x_31 = reshape(shape = var_1528, x = xt_3)[name = tensor("x_31")]; + tensor var_1530_perm_0 = const()[name = tensor("op_1530_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1534 = const()[name = tensor("op_1534"), val = tensor([12, 3, 256])]; + tensor var_1530 = transpose(perm = var_1530_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1534, x = var_1530)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 12, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1731,120 +1428,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_1970 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_1971 = const()[name = tensor("op_1971"), val = tensor([12, 3, 4, 64])]; - tensor var_1972 = reshape(shape = var_1971, x = var_1970)[name = tensor("op_1972")]; + tensor var_1542 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1543 = const()[name = tensor("op_1543"), val = tensor([12, 3, 4, 64])]; + tensor var_1544 = reshape(shape = var_1543, x = var_1542)[name = tensor("op_1544")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1976 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_1977 = const()[name = tensor("op_1977"), val = tensor(0x1p-3)]; - tensor var_1978 = mul(x = var_1976, y = var_1977)[name = tensor("op_1978")]; - tensor var_1979 = const()[name = tensor("op_1979"), val = tensor([12, 3, 4, 64])]; - tensor var_1980 = reshape(shape = var_1979, x = var_1978)[name = tensor("op_1980")]; + tensor var_1548 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1549 = const()[name = tensor("op_1549"), val = tensor(0x1p-3)]; + tensor var_1550 = mul(x = var_1548, y = var_1549)[name = tensor("op_1550")]; + tensor var_1551 = const()[name = tensor("op_1551"), val = tensor([12, 3, 4, 64])]; + tensor var_1552 = reshape(shape = var_1551, x = var_1550)[name = tensor("op_1552")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1984 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_1985 = const()[name = tensor("op_1985"), val = tensor([12, 3, 4, 64])]; - tensor var_1986 = reshape(shape = var_1985, x = var_1984)[name = tensor("op_1986")]; + tensor var_1556 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1557 = const()[name = tensor("op_1557"), val = tensor([12, 3, 4, 64])]; + tensor var_1558 = reshape(shape = var_1557, x = var_1556)[name = tensor("op_1558")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_267 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; + tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; - tensor const_40 = const()[name = tensor("const_40"), val = tensor(0x1.fffffep+127)]; - tensor clip_27 = clip(alpha = var_1698, beta = const_40, x = s_t)[name = tensor("clip_27")]; + tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; + tensor clip_27 = clip(alpha = var_1269, beta = const_32, x = s_t)[name = tensor("clip_27")]; tensor sqrt_s_t = sqrt(x = clip_27)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_1980)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_1972)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1552)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1544)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_2001 = const()[name = tensor("op_2001"), val = tensor([3, 1])]; - tensor var_2002 = reshape(shape = var_2001, x = sqrt_s_t)[name = tensor("op_2002")]; - tensor M = real_div(x = causal_with_valid_1, y = var_2002)[name = tensor("M")]; - tensor var_2004 = mul(x = qk, y = M)[name = tensor("op_2004")]; + tensor var_1573 = const()[name = tensor("op_1573"), val = tensor([3, 1])]; + tensor var_1574 = reshape(shape = var_1573, x = sqrt_s_t)[name = tensor("op_1574")]; + tensor M = real_div(x = causal_with_valid_1, y = var_1574)[name = tensor("M")]; + tensor var_1576 = mul(x = qk, y = M)[name = tensor("op_1576")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_1986)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_2004, y = v_17)[name = tensor("inner")]; - tensor var_2006_transpose_x_0 = const()[name = tensor("op_2006_transpose_x_0"), val = tensor(false)]; - tensor var_2006_transpose_y_0 = const()[name = tensor("op_2006_transpose_y_0"), val = tensor(false)]; - tensor var_2006 = matmul(transpose_x = var_2006_transpose_x_0, transpose_y = var_2006_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_2006")]; - tensor var_2007 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_2007")]; - tensor var_2008 = const()[name = tensor("op_2008"), val = tensor([1, 1, 3, 1])]; - tensor var_2009 = reshape(shape = var_2008, x = var_2007)[name = tensor("op_2009")]; - tensor cross = mul(x = var_2006, y = var_2009)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1558)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1576, y = v_17)[name = tensor("inner")]; + tensor var_1578_transpose_x_0 = const()[name = tensor("op_1578_transpose_x_0"), val = tensor(false)]; + tensor var_1578_transpose_y_0 = const()[name = tensor("op_1578_transpose_y_0"), val = tensor(false)]; + tensor var_1578 = matmul(transpose_x = var_1578_transpose_x_0, transpose_y = var_1578_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1578")]; + tensor var_1579 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1579")]; + tensor var_1580 = const()[name = tensor("op_1580"), val = tensor([1, 1, 3, 1])]; + tensor var_1581 = reshape(shape = var_1580, x = var_1579)[name = tensor("op_1581")]; + tensor cross = mul(x = var_1578, y = var_1581)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_1832)[name = tensor("v_masked")]; - tensor var_2015 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_2015")]; - tensor var_2017_transpose_x_1 = const()[name = tensor("op_2017_transpose_x_1"), val = tensor(true)]; - tensor var_2017_transpose_y_1 = const()[name = tensor("op_2017_transpose_y_1"), val = tensor(false)]; - tensor var_2017 = matmul(transpose_x = var_2017_transpose_x_1, transpose_y = var_2017_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_2017")]; - tensor new_kv_unnorm = add(x = var_2015, y = var_2017)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_1840)[name = tensor("new_scale")]; - tensor const_41 = const()[name = tensor("const_41"), val = tensor(0x1.fffffep+127)]; - tensor clip_28 = clip(alpha = var_1698, beta = const_41, x = new_scale)[name = tensor("clip_28")]; + tensor v_masked = mul(x = v_17, y = var_1404)[name = tensor("v_masked")]; + tensor var_1587 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1587")]; + tensor var_1589_transpose_x_1 = const()[name = tensor("op_1589_transpose_x_1"), val = tensor(true)]; + tensor var_1589_transpose_y_1 = const()[name = tensor("op_1589_transpose_y_1"), val = tensor(false)]; + tensor var_1589 = matmul(transpose_x = var_1589_transpose_x_1, transpose_y = var_1589_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1589")]; + tensor new_kv_unnorm = add(x = var_1587, y = var_1589)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1412)[name = tensor("new_scale")]; + tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; + tensor clip_28 = clip(alpha = var_1269, beta = const_33, x = new_scale)[name = tensor("clip_28")]; tensor sqrt_new_scale = sqrt(x = clip_28)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_2026_perm_0 = const()[name = tensor("op_2026_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1598_perm_0 = const()[name = tensor("op_1598_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_2026 = transpose(perm = var_2026_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1695, x = var_2026)[name = tensor("out_33")]; - tensor var_2030 = const()[name = tensor("op_2030"), val = tensor([12, 3, 256])]; - tensor out = reshape(shape = var_2030, x = out_33)[name = tensor("out")]; - tensor var_2032 = silu(x = input_267)[name = tensor("op_2032")]; - tensor input_269 = mul(x = var_2032, y = out)[name = tensor("input_269")]; - tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_269)[name = tensor("linear_51")]; - tensor input_271 = add(x = x, y = ret_out)[name = tensor("input_271")]; + tensor var_1598 = transpose(perm = var_1598_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1266, x = var_1598)[name = tensor("out_33")]; + tensor var_1602 = const()[name = tensor("op_1602"), val = tensor([12, 3, 256])]; + tensor out = reshape(shape = var_1602, x = out_33)[name = tensor("out")]; + tensor var_1604 = silu(x = input_187)[name = tensor("op_1604")]; + tensor input_189 = mul(x = var_1604, y = out)[name = tensor("input_189")]; + tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; + tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1693, gamma = decoder_norm11_1_weight, x = input_271)[name = tensor("xt_5")]; - tensor var_2042 = const()[name = tensor("op_2042"), val = tensor([1, 12, 3, 256])]; - tensor var_2043 = reshape(shape = var_2042, x = xt_5)[name = tensor("op_2043")]; - tensor var_2044_perm_0 = const()[name = tensor("op_2044_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2047 = const()[name = tensor("op_2047"), val = tensor([3, 12, 256])]; - tensor var_2044 = transpose(perm = var_2044_perm_0, x = var_2043)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_2047, x = var_2044)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1264, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1614 = const()[name = tensor("op_1614"), val = tensor([1, 12, 3, 256])]; + tensor var_1615 = reshape(shape = var_1614, x = xt_5)[name = tensor("op_1615")]; + tensor var_1616_perm_0 = const()[name = tensor("op_1616_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1619 = const()[name = tensor("op_1619"), val = tensor([3, 12, 256])]; + tensor var_1616 = transpose(perm = var_1616_perm_0, x = var_1615)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1619, x = var_1616)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_2070 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_13 = const()[name = tensor("concat_13"), val = tensor([12, 3, 3, 256])]; - tensor var_2072 = reshape(shape = concat_13, x = var_2070)[name = tensor("op_2072")]; - tensor var_2073_axes_0 = const()[name = tensor("op_2073_axes_0"), val = tensor([0])]; - tensor var_2073 = expand_dims(axes = var_2073_axes_0, x = var_2072)[name = tensor("op_2073")]; - tensor var_2074_perm_0 = const()[name = tensor("op_2074_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2075_axes_0 = const()[name = tensor("op_2075_axes_0"), val = tensor([-2])]; - tensor var_2074 = transpose(perm = var_2074_perm_0, x = var_2073)[name = tensor("transpose_8")]; - tensor var_2075 = squeeze(axes = var_2075_axes_0, x = var_2074)[name = tensor("op_2075")]; + tensor var_1642 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_14 = const()[name = tensor("concat_14"), val = tensor([12, 3, 3, 256])]; + tensor var_1644 = reshape(shape = concat_14, x = var_1642)[name = tensor("op_1644")]; + tensor var_1645_axes_0 = const()[name = tensor("op_1645_axes_0"), val = tensor([0])]; + tensor var_1645 = expand_dims(axes = var_1645_axes_0, x = var_1644)[name = tensor("op_1645")]; + tensor var_1646_perm_0 = const()[name = tensor("op_1646_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1647_axes_0 = const()[name = tensor("op_1647_axes_0"), val = tensor([-2])]; + tensor var_1646 = transpose(perm = var_1646_perm_0, x = var_1645)[name = tensor("transpose_8")]; + tensor var_1647 = squeeze(axes = var_1647_axes_0, x = var_1646)[name = tensor("op_1647")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 12, 3, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_2075)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1647)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 12, 3, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_2075)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1647)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 12, 3, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_2075)[name = tensor("v_19")]; - tensor var_2083 = const()[name = tensor("op_2083"), val = tensor([12, 12, 64])]; - tensor var_2084 = reshape(shape = var_2083, x = q_19)[name = tensor("op_2084")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1647)[name = tensor("v_19")]; + tensor var_1655 = const()[name = tensor("op_1655"), val = tensor([12, 12, 64])]; + tensor var_1656 = reshape(shape = var_1655, x = q_19)[name = tensor("op_1656")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2090 = const()[name = tensor("op_2090"), val = tensor([12, 12, 64])]; - tensor var_2091 = reshape(shape = var_2090, x = k_19)[name = tensor("op_2091")]; + tensor var_1662 = const()[name = tensor("op_1662"), val = tensor([12, 12, 64])]; + tensor var_1663 = reshape(shape = var_1662, x = k_19)[name = tensor("op_1663")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2097 = const()[name = tensor("op_2097"), val = tensor([12, 12, 64])]; - tensor var_2098 = reshape(shape = var_2097, x = v_19)[name = tensor("op_2098")]; + tensor var_1669 = const()[name = tensor("op_1669"), val = tensor([12, 12, 64])]; + tensor var_1670 = reshape(shape = var_1669, x = v_19)[name = tensor("op_1670")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2101 = const()[name = tensor("op_2101"), val = tensor([3, 4, 12, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_2084)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_2101, x = q_21)[name = tensor("q")]; - tensor var_2103 = const()[name = tensor("op_2103"), val = tensor([3, 4, 12, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_2091)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_2103, x = k_21)[name = tensor("k")]; - tensor var_2105 = const()[name = tensor("op_2105"), val = tensor([3, 4, 12, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_2098)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_2105, x = v_21)[name = tensor("v")]; + tensor var_1673 = const()[name = tensor("op_1673"), val = tensor([3, 4, 12, 64])]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1656)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1673, x = q_21)[name = tensor("q")]; + tensor var_1675 = const()[name = tensor("op_1675"), val = tensor([3, 4, 12, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1663)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1675, x = k_21)[name = tensor("k")]; + tensor var_1677 = const()[name = tensor("op_1677"), val = tensor([3, 4, 12, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1670)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1677, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -1855,50 +1552,50 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_2108 = const()[name = tensor("op_2108"), val = tensor([2, 0, 1, 3])]; - tensor var_2113 = const()[name = tensor("op_2113"), val = tensor([36, 256])]; - tensor var_2109 = transpose(perm = var_2108, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_2113, x = var_2109)[name = tensor("attn_output_11")]; + tensor var_1680 = const()[name = tensor("op_1680"), val = tensor([2, 0, 1, 3])]; + tensor var_1685 = const()[name = tensor("op_1685"), val = tensor([36, 256])]; + tensor var_1681 = transpose(perm = var_1680, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1685, x = var_1681)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_2117 = const()[name = tensor("op_2117"), val = tensor([12, 3, 256])]; - tensor attn_output = reshape(shape = var_2117, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1689 = const()[name = tensor("op_1689"), val = tensor([12, 3, 256])]; + tensor attn_output = reshape(shape = var_1689, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; - tensor input_273 = add(x = query_5, y = sa2_out)[name = tensor("input_273")]; - tensor input_275_axes_0 = const()[name = tensor("input_275_axes_0"), val = tensor([-1])]; - tensor input_275 = layer_norm(axes = input_275_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1693, gamma = decoder_norm21_1_weight, x = input_273)[name = tensor("input_275")]; - tensor input_277 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_275)[name = tensor("linear_54")]; - tensor input_279 = relu(x = input_277)[name = tensor("input_279")]; - tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_279)[name = tensor("linear_55")]; - tensor input_281 = add(x = input_275, y = ffn_out)[name = tensor("input_281")]; + tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; + tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1264, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; + tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; + tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; + tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1693, gamma = decoder_norm22_1_weight, x = input_281)[name = tensor("xt")]; - tensor var_2137 = const()[name = tensor("op_2137"), val = tensor([1, 3, 12, 256])]; - tensor input = reshape(shape = var_2137, x = xt)[name = tensor("input")]; - tensor var_2139 = const()[name = tensor("op_2139"), val = tensor([-1])]; - tensor var_2140 = reduce_l2_norm(axes = var_2139, keep_dims = var_1696, x = input)[name = tensor("op_2140")]; - tensor const_50 = const()[name = tensor("const_50"), val = tensor(0x1.fffffep+127)]; - tensor clip_29 = clip(alpha = var_1687, beta = const_50, x = var_2140)[name = tensor("clip_29")]; - tensor var_2142 = real_div(x = input, y = clip_29)[name = tensor("op_2142")]; - tensor concat_17 = const()[name = tensor("concat_17"), val = tensor([3, 1, 256])]; - tensor reshape_0 = reshape(shape = concat_17, x = emb)[name = tensor("reshape_0")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1264, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1709 = const()[name = tensor("op_1709"), val = tensor([1, 3, 12, 256])]; + tensor input = reshape(shape = var_1709, x = xt)[name = tensor("input")]; + tensor var_1711 = const()[name = tensor("op_1711"), val = tensor([-1])]; + tensor var_1712 = reduce_l2_norm(axes = var_1711, keep_dims = var_1267, x = input)[name = tensor("op_1712")]; + tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; + tensor clip_29 = clip(alpha = var_1259, beta = const_42, x = var_1712)[name = tensor("clip_29")]; + tensor var_1714 = real_div(x = input, y = clip_29)[name = tensor("op_1714")]; + tensor concat_18 = const()[name = tensor("concat_18"), val = tensor([3, 1, 256])]; + tensor reshape_0 = reshape(shape = concat_18, x = emb)[name = tensor("reshape_0")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_18 = const()[name = tensor("concat_18"), val = tensor([3, 256, 12])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_2142)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_18, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_19 = const()[name = tensor("concat_19"), val = tensor([3, 256, 12])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1714)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_19, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = reshape_0, y = reshape_1)[name = tensor("matmul_2")]; - tensor concat_21 = const()[name = tensor("concat_21"), val = tensor([1, 3, 12])]; - tensor reshape_2 = reshape(shape = concat_21, x = matmul_2)[name = tensor("reshape_2")]; + tensor concat_22 = const()[name = tensor("concat_22"), val = tensor([1, 3, 12])]; + tensor reshape_2 = reshape(shape = concat_22, x = matmul_2)[name = tensor("reshape_2")]; tensor output_begin_0 = const()[name = tensor("output_begin_0"), val = tensor([0, 0, 1])]; tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 3, 11])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = reshape_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_2146")]; - tensor var_2148_axis_0 = const()[name = tensor("op_2148_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_2148_axis_0, values = (var_1844, nkv))[name = tensor("op_2148")]; - tensor var_2150_axis_0 = const()[name = tensor("op_2150_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_2150_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_2150")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1718")]; + tensor var_1720_axis_0 = const()[name = tensor("op_1720_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1720_axis_0, values = (var_1416, nkv))[name = tensor("op_1720")]; + tensor var_1722_axis_0 = const()[name = tensor("op_1722_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1722_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1722")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/DIHARD III/ls_eend_dih3_300ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/DIHARD III/ls_eend_dih3_300ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index 5b3121d512e77445db71cc80593dc4c22b8ac3bb..7c1665d98e235ba76d5733c1de76b3d33f64dae6 100644 --- a/optimized/DIHARD III/ls_eend_dih3_300ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/DIHARD III/ls_eend_dih3_300ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0330369e13fd6c861c899bcf8d2cdbb9f53c27bbbb9a230837a9ae19e7247c48 -size 269884 +oid sha256:15743431167cf4ff61d6249241e41f78fdaad02dca01a96b6ae37ddeef462a5e +size 222335 diff --git a/optimized/DIHARD III/ls_eend_dih3_300ms.mlpackage/Manifest.json b/optimized/DIHARD III/ls_eend_dih3_300ms.mlpackage/Manifest.json index fde05d50d18795d5578a13b496ee864b1c60658f..cafec8f5453378549c8a19e640fefcfffc447508 100644 --- a/optimized/DIHARD III/ls_eend_dih3_300ms.mlpackage/Manifest.json +++ b/optimized/DIHARD III/ls_eend_dih3_300ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "16C5BA4B-BC29-40A3-AF2E-A9F5B2585D2F": { + "49517ECA-763F-4AF3-B1AB-967939DCA129": { "author": "com.apple.CoreML", "description": "CoreML Model Specification", "name": "model.mlmodel", "path": "com.apple.CoreML/model.mlmodel" }, - "974DB91A-6F3C-4161-A88A-054DAE1CA398": { + "5487DD46-EAF9-4D81-805C-4ED1F2CD1068": { "author": "com.apple.CoreML", "description": "CoreML Model Weights", "name": "weights", "path": "com.apple.CoreML/weights" } }, - "rootModelIdentifier": "16C5BA4B-BC29-40A3-AF2E-A9F5B2585D2F" + "rootModelIdentifier": "49517ECA-763F-4AF3-B1AB-967939DCA129" } diff --git a/optimized/DIHARD III/ls_eend_dih3_400ms.mlmodelc/analytics/coremldata.bin b/optimized/DIHARD III/ls_eend_dih3_400ms.mlmodelc/analytics/coremldata.bin index 79fb45351f3478b030eb9756686c3812735bb17a..c9f475df1650f8c969e7cc2f8dfcc54e1eb0d457 100644 --- a/optimized/DIHARD III/ls_eend_dih3_400ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/DIHARD III/ls_eend_dih3_400ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:24c411a28d96c8e37a7c3cf643ea968aa9f1a406c686700553db50dc8c2afac4 +oid sha256:3642e2d80c9b028cc632445e5b9cf7e49c663ae04d2345e81aa6b056dd2140bd size 243 diff --git a/optimized/DIHARD III/ls_eend_dih3_400ms.mlmodelc/coremldata.bin b/optimized/DIHARD III/ls_eend_dih3_400ms.mlmodelc/coremldata.bin index b24dfad8c9e280c97a0e3f23312d5d5c2f5d8526..d609dc69a8d432158b4a544c7f73d9d2adb1af00 100644 --- a/optimized/DIHARD III/ls_eend_dih3_400ms.mlmodelc/coremldata.bin +++ b/optimized/DIHARD III/ls_eend_dih3_400ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b4a207393d55b3f00a983806e2c0bba39ab7bc39bba6048e4b202bcb9c49e76d -size 1268 +oid sha256:6113b62011b897f53f33a55b0489c7706cf4a6867a87d43237bc4f3f37b20561 +size 1359 diff --git a/optimized/DIHARD III/ls_eend_dih3_400ms.mlmodelc/metadata.json b/optimized/DIHARD III/ls_eend_dih3_400ms.mlmodelc/metadata.json index f8dc552f03716f70e4dbaf99bf68567a15acaeaf..00eb72da8b6730ac4a5cc1d149e10a24971ec3a0 100644 --- a/optimized/DIHARD III/ls_eend_dih3_400ms.mlmodelc/metadata.json +++ b/optimized/DIHARD III/ls_eend_dih3_400ms.mlmodelc/metadata.json @@ -96,30 +96,30 @@ "Ios17.reshape" : 100, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 77, - "Split" : 16, + "Ios17.transpose" : 57, + "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 94, "Tile" : 17, - "Ios17.sliceByIndex" : 92, - "Ios16.sigmoid" : 17, + "Ios17.sliceByIndex" : 68, + "Ios16.sigmoid" : 5, "Ios16.reduceSum" : 1, "Ios17.squeeze" : 2, - "Ios17.layerNorm" : 45, - "Ios17.batchNorm" : 16, + "Ios17.layerNorm" : 33, + "Ios17.batchNorm" : 4, "Ios17.cast" : 32, "Ios17.sqrt" : 18, "Ios17.sub" : 32, - "Ios17.conv" : 49, + "Ios17.conv" : 13, "Ios17.realDiv" : 20, "Ios17.linear" : 56, "Ios17.concat" : 22, "Ios17.clip" : 38, "Ios17.greaterEqual" : 16, - "Ios16.silu" : 30, + "Ios16.silu" : 18, "Stack" : 6, "Ios16.cumsum" : 1, - "Ios17.mul" : 123 + "Ios17.mul" : 111 }, "computePrecision" : "Mixed (Float32, Int32)", "isUpdatable" : "0", @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"dih3\", \"model_label\": \"DIHARD III\", \"variant\": \"pipeline\", \"T\": 4, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"dih3\", \"model_label\": \"DIHARD III\", \"variant\": \"pipeline\", \"chunk_size\": 4, \"step_duration_ms\": 400, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/DIHARD III/ls_eend_dih3_400ms.mlmodelc/model.mil b/optimized/DIHARD III/ls_eend_dih3_400ms.mlmodelc/model.mil index ab2490e66dc769b71c76b7d12854b119a3d8d546..d04fbf3f7fb70564326a57bf225ae127afa5bbd7 100644 --- a/optimized/DIHARD III/ls_eend_dih3_400ms.mlmodelc/model.mil +++ b/optimized/DIHARD III/ls_eend_dih3_400ms.mlmodelc/model.mil @@ -217,6 +217,7 @@ program(1.0) tensor var_21 = const()[name = tensor("op_21"), val = tensor(0x0p+0)]; tensor var_24 = const()[name = tensor("op_24"), val = tensor(0x1.0c6f7ap-20)]; tensor var_27 = const()[name = tensor("op_27"), val = tensor(2)]; + tensor var_30 = const()[name = tensor("op_30"), val = tensor(0)]; tensor var_33 = const()[name = tensor("op_33"), val = tensor(1)]; tensor var_35 = const()[name = tensor("op_35"), val = tensor(0x1.4f8b58p-17)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(true)]; @@ -263,8 +264,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_77")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_78")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_180)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_172)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_196 = const()[name = tensor("op_196"), val = tensor([4, 1])]; tensor var_197 = reshape(shape = var_196, x = sqrt_s_t_1)[name = tensor("op_197")]; @@ -272,7 +273,7 @@ program(1.0) tensor var_199 = mul(x = qk_1, y = M_1)[name = tensor("op_199")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_76")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_186)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_199, y = v_1)[name = tensor("inner_1")]; tensor var_201_transpose_x_0 = const()[name = tensor("op_201_transpose_x_0"), val = tensor(false)]; tensor var_201_transpose_y_0 = const()[name = tensor("op_201_transpose_y_0"), val = tensor(false)]; @@ -293,7 +294,7 @@ program(1.0) tensor var_214 = real_div(x = new_kv_unnorm_1, y = var_213)[name = tensor("op_214")]; tensor var_215_perm_0 = const()[name = tensor("op_215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_75")]; + tensor var_215 = transpose(perm = var_215_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_24, x = var_215)[name = tensor("out_3")]; tensor var_219 = const()[name = tensor("op_219"), val = tensor([1, 4, 256])]; tensor out_5 = reshape(shape = var_219, x = out_3)[name = tensor("out_5")]; @@ -341,291 +342,176 @@ program(1.0) tensor var_252 = sub(x = var_17, y = use_roll_1)[name = tensor("op_252")]; tensor var_253 = mul(x = filled_window_1, y = var_252)[name = tensor("op_253")]; tensor var_254 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_254")]; - tensor input_21 = add(x = var_253, y = var_254)[name = tensor("input_21")]; + tensor window_3 = add(x = var_253, y = var_254)[name = tensor("window_3")]; tensor var_256 = const()[name = tensor("op_256"), val = tensor(0x1p+0)]; tensor var_257 = add(x = filled_1, y = var_256)[name = tensor("op_257")]; tensor clip_1 = clip(alpha = var_21, beta = var_16, x = var_257)[name = tensor("clip_1")]; + tensor var_260_begin_0 = const()[name = tensor("op_260_begin_0"), val = tensor([0, 1, 0])]; + tensor var_260_end_0 = const()[name = tensor("op_260_end_0"), val = tensor([1, 2, 256])]; + tensor var_260_end_mask_0 = const()[name = tensor("op_260_end_mask_0"), val = tensor([true, false, true])]; + tensor var_260 = slice_by_index(begin = var_260_begin_0, end = var_260_end_0, end_mask = var_260_end_mask_0, x = x_3)[name = tensor("op_260")]; + tensor var_263_begin_0 = const()[name = tensor("op_263_begin_0"), val = tensor([0, 1, 0])]; + tensor var_263_end_0 = const()[name = tensor("op_263_end_0"), val = tensor([1, 16, 256])]; + tensor var_263_end_mask_0 = const()[name = tensor("op_263_end_mask_0"), val = tensor([true, true, true])]; + tensor var_263 = slice_by_index(begin = var_263_begin_0, end = var_263_end_0, end_mask = var_263_end_mask_0, x = window_3)[name = tensor("op_263")]; + tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; + tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_263, var_260))[name = tensor("rolled_3")]; + tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; + tensor var_268 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_268")]; + tensor var_269_dtype_0 = const()[name = tensor("op_269_dtype_0"), val = tensor("fp32")]; + tensor var_270 = const()[name = tensor("op_270"), val = tensor([1, 16, 1])]; + tensor var_269 = cast(dtype = var_269_dtype_0, x = var_268)[name = tensor("cast_173")]; + tensor fill_mask_3 = reshape(shape = var_270, x = var_269)[name = tensor("fill_mask_3")]; + tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_260)[name = tensor("x_expanded_3")]; + tensor var_274 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_274")]; + tensor var_275 = mul(x = window_3, y = var_274)[name = tensor("op_275")]; + tensor var_276 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_276")]; + tensor filled_window_3 = add(x = var_275, y = var_276)[name = tensor("filled_window_3")]; + tensor var_278 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_278")]; + tensor var_279_dtype_0 = const()[name = tensor("op_279_dtype_0"), val = tensor("fp32")]; + tensor var_280 = const()[name = tensor("op_280"), val = tensor([1, 1, 1])]; + tensor var_279 = cast(dtype = var_279_dtype_0, x = var_278)[name = tensor("cast_172")]; + tensor use_roll_3 = reshape(shape = var_280, x = var_279)[name = tensor("use_roll_3")]; + tensor var_282 = sub(x = var_17, y = use_roll_3)[name = tensor("op_282")]; + tensor var_283 = mul(x = filled_window_3, y = var_282)[name = tensor("op_283")]; + tensor var_284 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_284")]; + tensor window_5 = add(x = var_283, y = var_284)[name = tensor("window_5")]; + tensor var_286 = const()[name = tensor("op_286"), val = tensor(0x1p+0)]; + tensor var_287 = add(x = clip_1, y = var_286)[name = tensor("op_287")]; + tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_287)[name = tensor("clip_3")]; + tensor var_290_begin_0 = const()[name = tensor("op_290_begin_0"), val = tensor([0, 2, 0])]; + tensor var_290_end_0 = const()[name = tensor("op_290_end_0"), val = tensor([1, 3, 256])]; + tensor var_290_end_mask_0 = const()[name = tensor("op_290_end_mask_0"), val = tensor([true, false, true])]; + tensor var_290 = slice_by_index(begin = var_290_begin_0, end = var_290_end_0, end_mask = var_290_end_mask_0, x = x_3)[name = tensor("op_290")]; + tensor var_293_begin_0 = const()[name = tensor("op_293_begin_0"), val = tensor([0, 1, 0])]; + tensor var_293_end_0 = const()[name = tensor("op_293_end_0"), val = tensor([1, 16, 256])]; + tensor var_293_end_mask_0 = const()[name = tensor("op_293_end_mask_0"), val = tensor([true, true, true])]; + tensor var_293 = slice_by_index(begin = var_293_begin_0, end = var_293_end_0, end_mask = var_293_end_mask_0, x = window_5)[name = tensor("op_293")]; + tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; + tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_293, var_290))[name = tensor("rolled_5")]; + tensor clip_4 = clip(alpha = var_21, beta = var_20, x = clip_3)[name = tensor("clip_4")]; + tensor var_298 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_298")]; + tensor var_299_dtype_0 = const()[name = tensor("op_299_dtype_0"), val = tensor("fp32")]; + tensor var_300 = const()[name = tensor("op_300"), val = tensor([1, 16, 1])]; + tensor var_299 = cast(dtype = var_299_dtype_0, x = var_298)[name = tensor("cast_171")]; + tensor fill_mask_5 = reshape(shape = var_300, x = var_299)[name = tensor("fill_mask_5")]; + tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_290)[name = tensor("x_expanded_5")]; + tensor var_304 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_304")]; + tensor var_305 = mul(x = window_5, y = var_304)[name = tensor("op_305")]; + tensor var_306 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_306")]; + tensor filled_window_5 = add(x = var_305, y = var_306)[name = tensor("filled_window_5")]; + tensor var_308 = greater_equal(x = clip_3, y = var_16)[name = tensor("op_308")]; + tensor var_309_dtype_0 = const()[name = tensor("op_309_dtype_0"), val = tensor("fp32")]; + tensor var_310 = const()[name = tensor("op_310"), val = tensor([1, 1, 1])]; + tensor var_309 = cast(dtype = var_309_dtype_0, x = var_308)[name = tensor("cast_170")]; + tensor use_roll_5 = reshape(shape = var_310, x = var_309)[name = tensor("use_roll_5")]; + tensor var_312 = sub(x = var_17, y = use_roll_5)[name = tensor("op_312")]; + tensor var_313 = mul(x = filled_window_5, y = var_312)[name = tensor("op_313")]; + tensor var_314 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_314")]; + tensor window_7 = add(x = var_313, y = var_314)[name = tensor("window_7")]; + tensor var_316 = const()[name = tensor("op_316"), val = tensor(0x1p+0)]; + tensor var_317 = add(x = clip_3, y = var_316)[name = tensor("op_317")]; + tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_317)[name = tensor("clip_5")]; + tensor var_320_begin_0 = const()[name = tensor("op_320_begin_0"), val = tensor([0, 3, 0])]; + tensor var_320_end_0 = const()[name = tensor("op_320_end_0"), val = tensor([1, 1, 256])]; + tensor var_320_end_mask_0 = const()[name = tensor("op_320_end_mask_0"), val = tensor([true, true, true])]; + tensor var_320 = slice_by_index(begin = var_320_begin_0, end = var_320_end_0, end_mask = var_320_end_mask_0, x = x_3)[name = tensor("op_320")]; + tensor var_323_begin_0 = const()[name = tensor("op_323_begin_0"), val = tensor([0, 1, 0])]; + tensor var_323_end_0 = const()[name = tensor("op_323_end_0"), val = tensor([1, 16, 256])]; + tensor var_323_end_mask_0 = const()[name = tensor("op_323_end_mask_0"), val = tensor([true, true, true])]; + tensor var_323 = slice_by_index(begin = var_323_begin_0, end = var_323_end_0, end_mask = var_323_end_mask_0, x = window_7)[name = tensor("op_323")]; + tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; + tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_323, var_320))[name = tensor("rolled_7")]; + tensor clip_6 = clip(alpha = var_21, beta = var_20, x = clip_5)[name = tensor("clip_6")]; + tensor var_328 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_328")]; + tensor var_329_dtype_0 = const()[name = tensor("op_329_dtype_0"), val = tensor("fp32")]; + tensor var_330 = const()[name = tensor("op_330"), val = tensor([1, 16, 1])]; + tensor var_329 = cast(dtype = var_329_dtype_0, x = var_328)[name = tensor("cast_169")]; + tensor fill_mask_7 = reshape(shape = var_330, x = var_329)[name = tensor("fill_mask_7")]; + tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_320)[name = tensor("x_expanded_7")]; + tensor var_334 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_334")]; + tensor var_335 = mul(x = window_7, y = var_334)[name = tensor("op_335")]; + tensor var_336 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_336")]; + tensor filled_window_7 = add(x = var_335, y = var_336)[name = tensor("filled_window_7")]; + tensor var_338 = greater_equal(x = clip_5, y = var_16)[name = tensor("op_338")]; + tensor var_339_dtype_0 = const()[name = tensor("op_339_dtype_0"), val = tensor("fp32")]; + tensor var_340 = const()[name = tensor("op_340"), val = tensor([1, 1, 1])]; + tensor var_339 = cast(dtype = var_339_dtype_0, x = var_338)[name = tensor("cast_168")]; + tensor use_roll_7 = reshape(shape = var_340, x = var_339)[name = tensor("use_roll_7")]; + tensor var_342 = sub(x = var_17, y = use_roll_7)[name = tensor("op_342")]; + tensor var_343 = mul(x = filled_window_7, y = var_342)[name = tensor("op_343")]; + tensor var_344 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_344")]; + tensor window_9 = add(x = var_343, y = var_344)[name = tensor("window_9")]; + tensor var_346 = const()[name = tensor("op_346"), val = tensor(0x1p+0)]; + tensor var_347 = add(x = clip_5, y = var_346)[name = tensor("op_347")]; + tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_347)[name = tensor("clip_7")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_30, interleave = input_21_interleave_0, values = (window_3, window_5, window_7, window_9))[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; - tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; + tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; tensor inputs_3_pad_type_0 = const()[name = tensor("inputs_3_pad_type_0"), val = tensor("valid")]; tensor inputs_3_strides_0 = const()[name = tensor("inputs_3_strides_0"), val = tensor([1])]; tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_74")]; - tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_278_split_sizes_0 = const()[name = tensor("op_278_split_sizes_0"), val = tensor([256, 256])]; - tensor var_278_axis_0 = const()[name = tensor("op_278_axis_0"), val = tensor(1)]; - tensor var_278_0, tensor var_278_1 = split(axis = var_278_axis_0, split_sizes = var_278_split_sizes_0, x = inputs_3)[name = tensor("op_278")]; - tensor var_280 = sigmoid(x = var_278_1)[name = tensor("op_280")]; - tensor inputs_5 = mul(x = var_278_0, y = var_280)[name = tensor("inputs_5")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; + tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; + tensor var_370_split_sizes_0 = const()[name = tensor("op_370_split_sizes_0"), val = tensor([256, 256])]; + tensor var_370_axis_0 = const()[name = tensor("op_370_axis_0"), val = tensor(1)]; + tensor var_370_0, tensor var_370_1 = split(axis = var_370_axis_0, split_sizes = var_370_split_sizes_0, x = inputs_3)[name = tensor("op_370")]; + tensor var_372 = sigmoid(x = var_370_1)[name = tensor("op_372")]; + tensor inputs_5 = mul(x = var_370_0, y = var_372)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; tensor outputs_aug_1_strides_0 = const()[name = tensor("outputs_aug_1_strides_0"), val = tensor([1])]; tensor outputs_aug_1_dilations_0 = const()[name = tensor("outputs_aug_1_dilations_0"), val = tensor([1])]; - tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; + tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; tensor input_25_begin_0 = const()[name = tensor("input_25_begin_0"), val = tensor([0, 0, 0])]; - tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([1, 256, 16])]; + tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([4, 256, 16])]; tensor input_25_end_mask_0 = const()[name = tensor("input_25_end_mask_0"), val = tensor([true, true, false])]; - tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; - tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; - tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; + tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; + tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; + tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; tensor input_29_pad_type_0 = const()[name = tensor("input_29_pad_type_0"), val = tensor("valid")]; tensor input_29_strides_0 = const()[name = tensor("input_29_strides_0"), val = tensor([1])]; tensor input_29_pad_0 = const()[name = tensor("input_29_pad_0"), val = tensor([0, 0])]; tensor input_29_dilations_0 = const()[name = tensor("input_29_dilations_0"), val = tensor([1])]; tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; - tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; + tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_311_begin_0 = const()[name = tensor("op_311_begin_0"), val = tensor([0, -1, 0])]; - tensor var_311_end_0 = const()[name = tensor("op_311_end_0"), val = tensor([1, 16, 256])]; - tensor var_311_end_mask_0 = const()[name = tensor("op_311_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_73")]; - tensor var_311 = slice_by_index(begin = var_311_begin_0, end = var_311_end_0, end_mask = var_311_end_mask_0, x = conv_out_1)[name = tensor("op_311")]; - tensor var_314_begin_0 = const()[name = tensor("op_314_begin_0"), val = tensor([0, 1, 0])]; - tensor var_314_end_0 = const()[name = tensor("op_314_end_0"), val = tensor([1, 2, 256])]; - tensor var_314_end_mask_0 = const()[name = tensor("op_314_end_mask_0"), val = tensor([true, false, true])]; - tensor var_314 = slice_by_index(begin = var_314_begin_0, end = var_314_end_0, end_mask = var_314_end_mask_0, x = x_3)[name = tensor("op_314")]; - tensor var_317_begin_0 = const()[name = tensor("op_317_begin_0"), val = tensor([0, 1, 0])]; - tensor var_317_end_0 = const()[name = tensor("op_317_end_0"), val = tensor([1, 16, 256])]; - tensor var_317_end_mask_0 = const()[name = tensor("op_317_end_mask_0"), val = tensor([true, true, true])]; - tensor var_317 = slice_by_index(begin = var_317_begin_0, end = var_317_end_0, end_mask = var_317_end_mask_0, x = input_21)[name = tensor("op_317")]; - tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_33, interleave = rolled_3_interleave_0, values = (var_317, var_314))[name = tensor("rolled_3")]; - tensor clip_2 = clip(alpha = var_21, beta = var_20, x = clip_1)[name = tensor("clip_2")]; - tensor var_322 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_322")]; - tensor var_323_dtype_0 = const()[name = tensor("op_323_dtype_0"), val = tensor("fp32")]; - tensor var_324 = const()[name = tensor("op_324"), val = tensor([1, 16, 1])]; - tensor var_323 = cast(dtype = var_323_dtype_0, x = var_322)[name = tensor("cast_173")]; - tensor fill_mask_3 = reshape(shape = var_324, x = var_323)[name = tensor("fill_mask_3")]; - tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_314)[name = tensor("x_expanded_3")]; - tensor var_328 = sub(x = var_17, y = fill_mask_3)[name = tensor("op_328")]; - tensor var_329 = mul(x = input_21, y = var_328)[name = tensor("op_329")]; - tensor var_330 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_330")]; - tensor filled_window_3 = add(x = var_329, y = var_330)[name = tensor("filled_window_3")]; - tensor var_332 = greater_equal(x = clip_1, y = var_16)[name = tensor("op_332")]; - tensor var_333_dtype_0 = const()[name = tensor("op_333_dtype_0"), val = tensor("fp32")]; - tensor var_334 = const()[name = tensor("op_334"), val = tensor([1, 1, 1])]; - tensor var_333 = cast(dtype = var_333_dtype_0, x = var_332)[name = tensor("cast_172")]; - tensor use_roll_3 = reshape(shape = var_334, x = var_333)[name = tensor("use_roll_3")]; - tensor var_336 = sub(x = var_17, y = use_roll_3)[name = tensor("op_336")]; - tensor var_337 = mul(x = filled_window_3, y = var_336)[name = tensor("op_337")]; - tensor var_338 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_338")]; - tensor input_31 = add(x = var_337, y = var_338)[name = tensor("input_31")]; - tensor var_340 = const()[name = tensor("op_340"), val = tensor(0x1p+0)]; - tensor var_341 = add(x = clip_1, y = var_340)[name = tensor("op_341")]; - tensor clip_3 = clip(alpha = var_21, beta = var_16, x = var_341)[name = tensor("clip_3")]; + tensor var_403_begin_0 = const()[name = tensor("op_403_begin_0"), val = tensor([0, -1, 0])]; + tensor var_403_end_0 = const()[name = tensor("op_403_end_0"), val = tensor([4, 16, 256])]; + tensor var_403_end_mask_0 = const()[name = tensor("op_403_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_403 = slice_by_index(begin = var_403_begin_0, end = var_403_end_0, end_mask = var_403_end_mask_0, x = conv_out_1)[name = tensor("op_403")]; + tensor var_405_perm_0 = const()[name = tensor("op_405_perm_0"), val = tensor([1, 0, 2])]; + tensor var_405 = transpose(perm = var_405_perm_0, x = var_403)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_405)[name = tensor("input_31")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; + tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; + tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; + tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; + tensor var_428 = const()[name = tensor("op_428"), val = tensor(0x1p-1)]; + tensor var_429 = mul(x = input_39, y = var_428)[name = tensor("op_429")]; + tensor input_41 = add(x = var_429, y = input_31)[name = tensor("input_41")]; + tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; + tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; + tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; + tensor input_45 = layer_norm(axes = input_45_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_43)[name = tensor("input_45")]; + tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; + tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; + tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; + tensor var_458 = const()[name = tensor("op_458"), val = tensor(0x1p-1)]; + tensor var_459 = mul(x = input_51, y = var_458)[name = tensor("op_459")]; + tensor input_53 = add(x = var_459, y = input_43)[name = tensor("input_53")]; tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; - tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_31)[name = tensor("x_7")]; - tensor input_33_perm_0 = const()[name = tensor("input_33_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_9_pad_type_0 = const()[name = tensor("inputs_9_pad_type_0"), val = tensor("valid")]; - tensor inputs_9_strides_0 = const()[name = tensor("inputs_9_strides_0"), val = tensor([1])]; - tensor inputs_9_pad_0 = const()[name = tensor("inputs_9_pad_0"), val = tensor([0, 0])]; - tensor inputs_9_dilations_0 = const()[name = tensor("inputs_9_dilations_0"), val = tensor([1])]; - tensor inputs_9_groups_0 = const()[name = tensor("inputs_9_groups_0"), val = tensor(1)]; - tensor input_33 = transpose(perm = input_33_perm_0, x = x_7)[name = tensor("transpose_72")]; - tensor inputs_9 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_9_dilations_0, groups = inputs_9_groups_0, pad = inputs_9_pad_0, pad_type = inputs_9_pad_type_0, strides = inputs_9_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_33)[name = tensor("inputs_9")]; - tensor var_362_split_sizes_0 = const()[name = tensor("op_362_split_sizes_0"), val = tensor([256, 256])]; - tensor var_362_axis_0 = const()[name = tensor("op_362_axis_0"), val = tensor(1)]; - tensor var_362_0, tensor var_362_1 = split(axis = var_362_axis_0, split_sizes = var_362_split_sizes_0, x = inputs_9)[name = tensor("op_362")]; - tensor var_364 = sigmoid(x = var_362_1)[name = tensor("op_364")]; - tensor inputs_11 = mul(x = var_362_0, y = var_364)[name = tensor("inputs_11")]; - tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; - tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; - tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; - tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_11)[name = tensor("outputs_aug_3")]; - tensor input_35_begin_0 = const()[name = tensor("input_35_begin_0"), val = tensor([0, 0, 0])]; - tensor input_35_end_0 = const()[name = tensor("input_35_end_0"), val = tensor([1, 256, 16])]; - tensor input_35_end_mask_0 = const()[name = tensor("input_35_end_mask_0"), val = tensor([true, true, false])]; - tensor input_35 = slice_by_index(begin = input_35_begin_0, end = input_35_end_0, end_mask = input_35_end_mask_0, x = outputs_aug_3)[name = tensor("input_35")]; - tensor inputs_13 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_35)[name = tensor("inputs_13")]; - tensor input_37 = silu(x = inputs_13)[name = tensor("input_37")]; - tensor input_39_pad_type_0 = const()[name = tensor("input_39_pad_type_0"), val = tensor("valid")]; - tensor input_39_strides_0 = const()[name = tensor("input_39_strides_0"), val = tensor([1])]; - tensor input_39_pad_0 = const()[name = tensor("input_39_pad_0"), val = tensor([0, 0])]; - tensor input_39_dilations_0 = const()[name = tensor("input_39_dilations_0"), val = tensor([1])]; - tensor input_39_groups_0 = const()[name = tensor("input_39_groups_0"), val = tensor(1)]; - tensor input_39 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_37)[name = tensor("input_39")]; - tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_395_begin_0 = const()[name = tensor("op_395_begin_0"), val = tensor([0, -1, 0])]; - tensor var_395_end_0 = const()[name = tensor("op_395_end_0"), val = tensor([1, 16, 256])]; - tensor var_395_end_mask_0 = const()[name = tensor("op_395_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_39)[name = tensor("transpose_71")]; - tensor var_395 = slice_by_index(begin = var_395_begin_0, end = var_395_end_0, end_mask = var_395_end_mask_0, x = conv_out_3)[name = tensor("op_395")]; - tensor var_398_begin_0 = const()[name = tensor("op_398_begin_0"), val = tensor([0, 2, 0])]; - tensor var_398_end_0 = const()[name = tensor("op_398_end_0"), val = tensor([1, 3, 256])]; - tensor var_398_end_mask_0 = const()[name = tensor("op_398_end_mask_0"), val = tensor([true, false, true])]; - tensor var_398 = slice_by_index(begin = var_398_begin_0, end = var_398_end_0, end_mask = var_398_end_mask_0, x = x_3)[name = tensor("op_398")]; - tensor var_401_begin_0 = const()[name = tensor("op_401_begin_0"), val = tensor([0, 1, 0])]; - tensor var_401_end_0 = const()[name = tensor("op_401_end_0"), val = tensor([1, 16, 256])]; - tensor var_401_end_mask_0 = const()[name = tensor("op_401_end_mask_0"), val = tensor([true, true, true])]; - tensor var_401 = slice_by_index(begin = var_401_begin_0, end = var_401_end_0, end_mask = var_401_end_mask_0, x = input_31)[name = tensor("op_401")]; - tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_33, interleave = rolled_5_interleave_0, values = (var_401, var_398))[name = tensor("rolled_5")]; - tensor clip_4 = clip(alpha = var_21, beta = var_20, x = clip_3)[name = tensor("clip_4")]; - tensor var_406 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_406")]; - tensor var_407_dtype_0 = const()[name = tensor("op_407_dtype_0"), val = tensor("fp32")]; - tensor var_408 = const()[name = tensor("op_408"), val = tensor([1, 16, 1])]; - tensor var_407 = cast(dtype = var_407_dtype_0, x = var_406)[name = tensor("cast_171")]; - tensor fill_mask_5 = reshape(shape = var_408, x = var_407)[name = tensor("fill_mask_5")]; - tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_398)[name = tensor("x_expanded_5")]; - tensor var_412 = sub(x = var_17, y = fill_mask_5)[name = tensor("op_412")]; - tensor var_413 = mul(x = input_31, y = var_412)[name = tensor("op_413")]; - tensor var_414 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_414")]; - tensor filled_window_5 = add(x = var_413, y = var_414)[name = tensor("filled_window_5")]; - tensor var_416 = greater_equal(x = clip_3, y = var_16)[name = tensor("op_416")]; - tensor var_417_dtype_0 = const()[name = tensor("op_417_dtype_0"), val = tensor("fp32")]; - tensor var_418 = const()[name = tensor("op_418"), val = tensor([1, 1, 1])]; - tensor var_417 = cast(dtype = var_417_dtype_0, x = var_416)[name = tensor("cast_170")]; - tensor use_roll_5 = reshape(shape = var_418, x = var_417)[name = tensor("use_roll_5")]; - tensor var_420 = sub(x = var_17, y = use_roll_5)[name = tensor("op_420")]; - tensor var_421 = mul(x = filled_window_5, y = var_420)[name = tensor("op_421")]; - tensor var_422 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_422")]; - tensor input_41 = add(x = var_421, y = var_422)[name = tensor("input_41")]; - tensor var_424 = const()[name = tensor("op_424"), val = tensor(0x1p+0)]; - tensor var_425 = add(x = clip_3, y = var_424)[name = tensor("op_425")]; - tensor clip_5 = clip(alpha = var_21, beta = var_16, x = var_425)[name = tensor("clip_5")]; - tensor x_9_axes_0 = const()[name = tensor("x_9_axes_0"), val = tensor([-1])]; - tensor x_9 = layer_norm(axes = x_9_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_41)[name = tensor("x_9")]; - tensor input_43_perm_0 = const()[name = tensor("input_43_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_15_pad_type_0 = const()[name = tensor("inputs_15_pad_type_0"), val = tensor("valid")]; - tensor inputs_15_strides_0 = const()[name = tensor("inputs_15_strides_0"), val = tensor([1])]; - tensor inputs_15_pad_0 = const()[name = tensor("inputs_15_pad_0"), val = tensor([0, 0])]; - tensor inputs_15_dilations_0 = const()[name = tensor("inputs_15_dilations_0"), val = tensor([1])]; - tensor inputs_15_groups_0 = const()[name = tensor("inputs_15_groups_0"), val = tensor(1)]; - tensor input_43 = transpose(perm = input_43_perm_0, x = x_9)[name = tensor("transpose_70")]; - tensor inputs_15 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_15_dilations_0, groups = inputs_15_groups_0, pad = inputs_15_pad_0, pad_type = inputs_15_pad_type_0, strides = inputs_15_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_43)[name = tensor("inputs_15")]; - tensor var_446_split_sizes_0 = const()[name = tensor("op_446_split_sizes_0"), val = tensor([256, 256])]; - tensor var_446_axis_0 = const()[name = tensor("op_446_axis_0"), val = tensor(1)]; - tensor var_446_0, tensor var_446_1 = split(axis = var_446_axis_0, split_sizes = var_446_split_sizes_0, x = inputs_15)[name = tensor("op_446")]; - tensor var_448 = sigmoid(x = var_446_1)[name = tensor("op_448")]; - tensor inputs_17 = mul(x = var_446_0, y = var_448)[name = tensor("inputs_17")]; - tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; - tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; - tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; - tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_17)[name = tensor("outputs_aug_5")]; - tensor input_45_begin_0 = const()[name = tensor("input_45_begin_0"), val = tensor([0, 0, 0])]; - tensor input_45_end_0 = const()[name = tensor("input_45_end_0"), val = tensor([1, 256, 16])]; - tensor input_45_end_mask_0 = const()[name = tensor("input_45_end_mask_0"), val = tensor([true, true, false])]; - tensor input_45 = slice_by_index(begin = input_45_begin_0, end = input_45_end_0, end_mask = input_45_end_mask_0, x = outputs_aug_5)[name = tensor("input_45")]; - tensor inputs_19 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_45)[name = tensor("inputs_19")]; - tensor input_47 = silu(x = inputs_19)[name = tensor("input_47")]; - tensor input_49_pad_type_0 = const()[name = tensor("input_49_pad_type_0"), val = tensor("valid")]; - tensor input_49_strides_0 = const()[name = tensor("input_49_strides_0"), val = tensor([1])]; - tensor input_49_pad_0 = const()[name = tensor("input_49_pad_0"), val = tensor([0, 0])]; - tensor input_49_dilations_0 = const()[name = tensor("input_49_dilations_0"), val = tensor([1])]; - tensor input_49_groups_0 = const()[name = tensor("input_49_groups_0"), val = tensor(1)]; - tensor input_49 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_49_dilations_0, groups = input_49_groups_0, pad = input_49_pad_0, pad_type = input_49_pad_type_0, strides = input_49_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_47)[name = tensor("input_49")]; - tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_479_begin_0 = const()[name = tensor("op_479_begin_0"), val = tensor([0, -1, 0])]; - tensor var_479_end_0 = const()[name = tensor("op_479_end_0"), val = tensor([1, 16, 256])]; - tensor var_479_end_mask_0 = const()[name = tensor("op_479_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_49)[name = tensor("transpose_69")]; - tensor var_479 = slice_by_index(begin = var_479_begin_0, end = var_479_end_0, end_mask = var_479_end_mask_0, x = conv_out_5)[name = tensor("op_479")]; - tensor var_482_begin_0 = const()[name = tensor("op_482_begin_0"), val = tensor([0, 3, 0])]; - tensor var_482_end_0 = const()[name = tensor("op_482_end_0"), val = tensor([1, 1, 256])]; - tensor var_482_end_mask_0 = const()[name = tensor("op_482_end_mask_0"), val = tensor([true, true, true])]; - tensor var_482 = slice_by_index(begin = var_482_begin_0, end = var_482_end_0, end_mask = var_482_end_mask_0, x = x_3)[name = tensor("op_482")]; - tensor var_485_begin_0 = const()[name = tensor("op_485_begin_0"), val = tensor([0, 1, 0])]; - tensor var_485_end_0 = const()[name = tensor("op_485_end_0"), val = tensor([1, 16, 256])]; - tensor var_485_end_mask_0 = const()[name = tensor("op_485_end_mask_0"), val = tensor([true, true, true])]; - tensor var_485 = slice_by_index(begin = var_485_begin_0, end = var_485_end_0, end_mask = var_485_end_mask_0, x = input_41)[name = tensor("op_485")]; - tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; - tensor rolled_7 = concat(axis = var_33, interleave = rolled_7_interleave_0, values = (var_485, var_482))[name = tensor("rolled_7")]; - tensor clip_6 = clip(alpha = var_21, beta = var_20, x = clip_5)[name = tensor("clip_6")]; - tensor var_490 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_490")]; - tensor var_491_dtype_0 = const()[name = tensor("op_491_dtype_0"), val = tensor("fp32")]; - tensor var_492 = const()[name = tensor("op_492"), val = tensor([1, 16, 1])]; - tensor var_491 = cast(dtype = var_491_dtype_0, x = var_490)[name = tensor("cast_169")]; - tensor fill_mask_7 = reshape(shape = var_492, x = var_491)[name = tensor("fill_mask_7")]; - tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_482)[name = tensor("x_expanded_7")]; - tensor var_496 = sub(x = var_17, y = fill_mask_7)[name = tensor("op_496")]; - tensor var_497 = mul(x = input_41, y = var_496)[name = tensor("op_497")]; - tensor var_498 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_498")]; - tensor filled_window_7 = add(x = var_497, y = var_498)[name = tensor("filled_window_7")]; - tensor var_500 = greater_equal(x = clip_5, y = var_16)[name = tensor("op_500")]; - tensor var_501_dtype_0 = const()[name = tensor("op_501_dtype_0"), val = tensor("fp32")]; - tensor var_502 = const()[name = tensor("op_502"), val = tensor([1, 1, 1])]; - tensor var_501 = cast(dtype = var_501_dtype_0, x = var_500)[name = tensor("cast_168")]; - tensor use_roll_7 = reshape(shape = var_502, x = var_501)[name = tensor("use_roll_7")]; - tensor var_504 = sub(x = var_17, y = use_roll_7)[name = tensor("op_504")]; - tensor var_505 = mul(x = filled_window_7, y = var_504)[name = tensor("op_505")]; - tensor var_506 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_506")]; - tensor input_51 = add(x = var_505, y = var_506)[name = tensor("input_51")]; - tensor var_508 = const()[name = tensor("op_508"), val = tensor(0x1p+0)]; - tensor var_509 = add(x = clip_5, y = var_508)[name = tensor("op_509")]; - tensor clip_7 = clip(alpha = var_21, beta = var_16, x = var_509)[name = tensor("clip_7")]; - tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; - tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_0_weight, x = input_51)[name = tensor("x_11")]; - tensor input_53_perm_0 = const()[name = tensor("input_53_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_21_pad_type_0 = const()[name = tensor("inputs_21_pad_type_0"), val = tensor("valid")]; - tensor inputs_21_strides_0 = const()[name = tensor("inputs_21_strides_0"), val = tensor([1])]; - tensor inputs_21_pad_0 = const()[name = tensor("inputs_21_pad_0"), val = tensor([0, 0])]; - tensor inputs_21_dilations_0 = const()[name = tensor("inputs_21_dilations_0"), val = tensor([1])]; - tensor inputs_21_groups_0 = const()[name = tensor("inputs_21_groups_0"), val = tensor(1)]; - tensor input_53 = transpose(perm = input_53_perm_0, x = x_11)[name = tensor("transpose_68")]; - tensor inputs_21 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_21_dilations_0, groups = inputs_21_groups_0, pad = inputs_21_pad_0, pad_type = inputs_21_pad_type_0, strides = inputs_21_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_53)[name = tensor("inputs_21")]; - tensor var_530_split_sizes_0 = const()[name = tensor("op_530_split_sizes_0"), val = tensor([256, 256])]; - tensor var_530_axis_0 = const()[name = tensor("op_530_axis_0"), val = tensor(1)]; - tensor var_530_0, tensor var_530_1 = split(axis = var_530_axis_0, split_sizes = var_530_split_sizes_0, x = inputs_21)[name = tensor("op_530")]; - tensor var_532 = sigmoid(x = var_530_1)[name = tensor("op_532")]; - tensor inputs_23 = mul(x = var_530_0, y = var_532)[name = tensor("inputs_23")]; - tensor outputs_aug_7_pad_type_0 = const()[name = tensor("outputs_aug_7_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_7_pad_0 = const()[name = tensor("outputs_aug_7_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_7_groups_0 = const()[name = tensor("outputs_aug_7_groups_0"), val = tensor(256)]; - tensor outputs_aug_7_strides_0 = const()[name = tensor("outputs_aug_7_strides_0"), val = tensor([1])]; - tensor outputs_aug_7_dilations_0 = const()[name = tensor("outputs_aug_7_dilations_0"), val = tensor([1])]; - tensor outputs_aug_7 = conv(dilations = outputs_aug_7_dilations_0, groups = outputs_aug_7_groups_0, pad = outputs_aug_7_pad_0, pad_type = outputs_aug_7_pad_type_0, strides = outputs_aug_7_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_23)[name = tensor("outputs_aug_7")]; - tensor input_55_begin_0 = const()[name = tensor("input_55_begin_0"), val = tensor([0, 0, 0])]; - tensor input_55_end_0 = const()[name = tensor("input_55_end_0"), val = tensor([1, 256, 16])]; - tensor input_55_end_mask_0 = const()[name = tensor("input_55_end_mask_0"), val = tensor([true, true, false])]; - tensor input_55 = slice_by_index(begin = input_55_begin_0, end = input_55_end_0, end_mask = input_55_end_mask_0, x = outputs_aug_7)[name = tensor("input_55")]; - tensor inputs_25 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_55)[name = tensor("inputs_25")]; - tensor input_57 = silu(x = inputs_25)[name = tensor("input_57")]; - tensor input_59_pad_type_0 = const()[name = tensor("input_59_pad_type_0"), val = tensor("valid")]; - tensor input_59_strides_0 = const()[name = tensor("input_59_strides_0"), val = tensor([1])]; - tensor input_59_pad_0 = const()[name = tensor("input_59_pad_0"), val = tensor([0, 0])]; - tensor input_59_dilations_0 = const()[name = tensor("input_59_dilations_0"), val = tensor([1])]; - tensor input_59_groups_0 = const()[name = tensor("input_59_groups_0"), val = tensor(1)]; - tensor input_59 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_59_dilations_0, groups = input_59_groups_0, pad = input_59_pad_0, pad_type = input_59_pad_type_0, strides = input_59_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_57)[name = tensor("input_59")]; - tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_563_begin_0 = const()[name = tensor("op_563_begin_0"), val = tensor([0, -1, 0])]; - tensor var_563_end_0 = const()[name = tensor("op_563_end_0"), val = tensor([1, 16, 256])]; - tensor var_563_end_mask_0 = const()[name = tensor("op_563_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_59)[name = tensor("transpose_67")]; - tensor var_563 = slice_by_index(begin = var_563_begin_0, end = var_563_end_0, end_mask = var_563_end_mask_0, x = conv_out_7)[name = tensor("op_563")]; - tensor var_566_interleave_0 = const()[name = tensor("op_566_interleave_0"), val = tensor(false)]; - tensor var_566 = concat(axis = var_33, interleave = var_566_interleave_0, values = (var_311, var_395, var_479, var_563))[name = tensor("op_566")]; - tensor input_61 = add(x = x_3, y = var_566)[name = tensor("input_61")]; - tensor input_63_axes_0 = const()[name = tensor("input_63_axes_0"), val = tensor([-1])]; - tensor input_63 = layer_norm(axes = input_63_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_61)[name = tensor("input_63")]; - tensor inputs_27 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_63)[name = tensor("linear_8")]; - tensor input_65 = silu(x = inputs_27)[name = tensor("input_65")]; - tensor input_69 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_65)[name = tensor("linear_9")]; - tensor var_589 = const()[name = tensor("op_589"), val = tensor(0x1p-1)]; - tensor var_590 = mul(x = input_69, y = var_589)[name = tensor("op_590")]; - tensor input_71 = add(x = var_590, y = input_61)[name = tensor("input_71")]; - tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; - tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_35, gamma = encoder_layer_norm_0_weight, x = input_71)[name = tensor("input_73")]; - tensor input_75_axes_0 = const()[name = tensor("input_75_axes_0"), val = tensor([-1])]; - tensor input_75 = layer_norm(axes = input_75_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_73)[name = tensor("input_75")]; - tensor inputs_29 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_75)[name = tensor("linear_10")]; - tensor input_77 = silu(x = inputs_29)[name = tensor("input_77")]; - tensor input_81 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_77)[name = tensor("linear_11")]; - tensor var_619 = const()[name = tensor("op_619"), val = tensor(0x1p-1)]; - tensor var_620 = mul(x = input_81, y = var_619)[name = tensor("op_620")]; - tensor input_83 = add(x = var_620, y = input_73)[name = tensor("input_83")]; - tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; - tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_83)[name = tensor("x_13")]; + tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_35, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_3_end_0 = const()[name = tensor("prev_kv_3_end_0"), val = tensor([2, 1, 4, 64, 64])]; tensor prev_kv_3_end_mask_0 = const()[name = tensor("prev_kv_3_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -636,389 +522,274 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_634 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_13)[name = tensor("linear_12")]; - tensor var_635 = const()[name = tensor("op_635"), val = tensor([1, 4, 4, 64])]; - tensor var_636 = reshape(shape = var_635, x = var_634)[name = tensor("op_636")]; + tensor var_473 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_474 = const()[name = tensor("op_474"), val = tensor([1, 4, 4, 64])]; + tensor var_475 = reshape(shape = var_474, x = var_473)[name = tensor("op_475")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_640 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_13)[name = tensor("linear_13")]; - tensor var_641 = const()[name = tensor("op_641"), val = tensor(0x1p-3)]; - tensor var_642 = mul(x = var_640, y = var_641)[name = tensor("op_642")]; - tensor var_643 = const()[name = tensor("op_643"), val = tensor([1, 4, 4, 64])]; - tensor var_644 = reshape(shape = var_643, x = var_642)[name = tensor("op_644")]; + tensor var_479 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_480 = const()[name = tensor("op_480"), val = tensor(0x1p-3)]; + tensor var_481 = mul(x = var_479, y = var_480)[name = tensor("op_481")]; + tensor var_482 = const()[name = tensor("op_482"), val = tensor([1, 4, 4, 64])]; + tensor var_483 = reshape(shape = var_482, x = var_481)[name = tensor("op_483")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_648 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_13)[name = tensor("linear_14")]; - tensor var_649 = const()[name = tensor("op_649"), val = tensor([1, 4, 4, 64])]; - tensor var_650 = reshape(shape = var_649, x = var_648)[name = tensor("op_650")]; + tensor var_487 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_488 = const()[name = tensor("op_488"), val = tensor([1, 4, 4, 64])]; + tensor var_489 = reshape(shape = var_488, x = var_487)[name = tensor("op_489")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_87 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_13)[name = tensor("linear_15")]; + tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; tensor s_t_3 = add(x = prev_scale_3, y = encoder__t_index)[name = tensor("s_t_3")]; tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_644)[name = tensor("transpose_65")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_636)[name = tensor("transpose_66")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_483)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_475)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_660 = const()[name = tensor("op_660"), val = tensor([4, 1])]; - tensor var_661 = reshape(shape = var_660, x = sqrt_s_t_3)[name = tensor("op_661")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_661)[name = tensor("M_3")]; - tensor var_663 = mul(x = qk_3, y = M_3)[name = tensor("op_663")]; + tensor var_499 = const()[name = tensor("op_499"), val = tensor([4, 1])]; + tensor var_500 = reshape(shape = var_499, x = sqrt_s_t_3)[name = tensor("op_500")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_500)[name = tensor("M_3")]; + tensor var_502 = mul(x = qk_3, y = M_3)[name = tensor("op_502")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_650)[name = tensor("transpose_64")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_663, y = v_3)[name = tensor("inner_3")]; - tensor var_665_transpose_x_0 = const()[name = tensor("op_665_transpose_x_0"), val = tensor(false)]; - tensor var_665_transpose_y_0 = const()[name = tensor("op_665_transpose_y_0"), val = tensor(false)]; - tensor var_665 = matmul(transpose_x = var_665_transpose_x_0, transpose_y = var_665_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_665")]; - tensor var_666 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_666")]; - tensor var_667 = const()[name = tensor("op_667"), val = tensor([1, 1, 4, 1])]; - tensor var_668 = reshape(shape = var_667, x = var_666)[name = tensor("op_668")]; - tensor cross_3 = mul(x = var_665, y = var_668)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_489)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_502, y = v_3)[name = tensor("inner_3")]; + tensor var_504_transpose_x_0 = const()[name = tensor("op_504_transpose_x_0"), val = tensor(false)]; + tensor var_504_transpose_y_0 = const()[name = tensor("op_504_transpose_y_0"), val = tensor(false)]; + tensor var_504 = matmul(transpose_x = var_504_transpose_x_0, transpose_y = var_504_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_504")]; + tensor var_505 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_505")]; + tensor var_506 = const()[name = tensor("op_506"), val = tensor([1, 1, 4, 1])]; + tensor var_507 = reshape(shape = var_506, x = var_505)[name = tensor("op_507")]; + tensor cross_3 = mul(x = var_504, y = var_507)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_671 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_671")]; - tensor var_673_transpose_x_1 = const()[name = tensor("op_673_transpose_x_1"), val = tensor(true)]; - tensor var_673_transpose_y_1 = const()[name = tensor("op_673_transpose_y_1"), val = tensor(false)]; - tensor var_673 = matmul(transpose_x = var_673_transpose_x_1, transpose_y = var_673_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_673")]; - tensor new_kv_unnorm_3 = add(x = var_671, y = var_673)[name = tensor("new_kv_unnorm_3")]; - tensor var_675 = const()[name = tensor("op_675"), val = tensor(0x1p+2)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_675)[name = tensor("new_scale_3")]; - tensor var_677 = sqrt(x = new_scale_3)[name = tensor("op_677")]; - tensor var_678 = real_div(x = new_kv_unnorm_3, y = var_677)[name = tensor("op_678")]; - tensor var_679_perm_0 = const()[name = tensor("op_679_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_510 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_510")]; + tensor var_512_transpose_x_1 = const()[name = tensor("op_512_transpose_x_1"), val = tensor(true)]; + tensor var_512_transpose_y_1 = const()[name = tensor("op_512_transpose_y_1"), val = tensor(false)]; + tensor var_512 = matmul(transpose_x = var_512_transpose_x_1, transpose_y = var_512_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_512")]; + tensor new_kv_unnorm_3 = add(x = var_510, y = var_512)[name = tensor("new_kv_unnorm_3")]; + tensor var_514 = const()[name = tensor("op_514"), val = tensor(0x1p+2)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_514)[name = tensor("new_scale_3")]; + tensor var_516 = sqrt(x = new_scale_3)[name = tensor("op_516")]; + tensor var_517 = real_div(x = new_kv_unnorm_3, y = var_516)[name = tensor("op_517")]; + tensor var_518_perm_0 = const()[name = tensor("op_518_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_679 = transpose(perm = var_679_perm_0, x = out_7)[name = tensor("transpose_63")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_679)[name = tensor("out_9")]; - tensor var_683 = const()[name = tensor("op_683"), val = tensor([1, 4, 256])]; - tensor out_11 = reshape(shape = var_683, x = out_9)[name = tensor("out_11")]; - tensor var_685 = silu(x = input_87)[name = tensor("op_685")]; - tensor input_89 = mul(x = var_685, y = out_11)[name = tensor("input_89")]; - tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_89)[name = tensor("linear_16")]; - tensor x_15 = add(x = input_83, y = ret_out_3)[name = tensor("x_15")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor var_518 = transpose(perm = var_518_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_24, x = var_518)[name = tensor("out_9")]; + tensor var_522 = const()[name = tensor("op_522"), val = tensor([1, 4, 256])]; + tensor out_11 = reshape(shape = var_522, x = out_9)[name = tensor("out_11")]; + tensor var_524 = silu(x = input_57)[name = tensor("op_524")]; + tensor input_59 = mul(x = var_524, y = out_11)[name = tensor("input_59")]; + tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; + tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; + tensor window_11_begin_0 = const()[name = tensor("window_11_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_11_end_0 = const()[name = tensor("window_11_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_11_end_mask_0 = const()[name = tensor("window_11_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_11_squeeze_mask_0 = const()[name = tensor("window_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_11 = slice_by_index(begin = window_11_begin_0, end = window_11_end_0, end_mask = window_11_end_mask_0, squeeze_mask = window_11_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_11")]; tensor filled_9_begin_0 = const()[name = tensor("filled_9_begin_0"), val = tensor([1, 0])]; tensor filled_9_end_0 = const()[name = tensor("filled_9_end_0"), val = tensor([2, 1])]; tensor filled_9_end_mask_0 = const()[name = tensor("filled_9_end_mask_0"), val = tensor([false, true])]; tensor filled_9_squeeze_mask_0 = const()[name = tensor("filled_9_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_9 = slice_by_index(begin = filled_9_begin_0, end = filled_9_end_0, end_mask = filled_9_end_mask_0, squeeze_mask = filled_9_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_9")]; - tensor var_694_begin_0 = const()[name = tensor("op_694_begin_0"), val = tensor([0, 0, 0])]; - tensor var_694_end_0 = const()[name = tensor("op_694_end_0"), val = tensor([1, 1, 256])]; - tensor var_694_end_mask_0 = const()[name = tensor("op_694_end_mask_0"), val = tensor([true, false, true])]; - tensor var_694 = slice_by_index(begin = var_694_begin_0, end = var_694_end_0, end_mask = var_694_end_mask_0, x = x_15)[name = tensor("op_694")]; - tensor var_697_begin_0 = const()[name = tensor("op_697_begin_0"), val = tensor([0, 1, 0])]; - tensor var_697_end_0 = const()[name = tensor("op_697_end_0"), val = tensor([1, 16, 256])]; - tensor var_697_end_mask_0 = const()[name = tensor("op_697_end_mask_0"), val = tensor([true, true, true])]; - tensor var_697 = slice_by_index(begin = var_697_begin_0, end = var_697_end_0, end_mask = var_697_end_mask_0, x = window_3)[name = tensor("op_697")]; + tensor var_533_begin_0 = const()[name = tensor("op_533_begin_0"), val = tensor([0, 0, 0])]; + tensor var_533_end_0 = const()[name = tensor("op_533_end_0"), val = tensor([1, 1, 256])]; + tensor var_533_end_mask_0 = const()[name = tensor("op_533_end_mask_0"), val = tensor([true, false, true])]; + tensor var_533 = slice_by_index(begin = var_533_begin_0, end = var_533_end_0, end_mask = var_533_end_mask_0, x = x_9)[name = tensor("op_533")]; + tensor var_536_begin_0 = const()[name = tensor("op_536_begin_0"), val = tensor([0, 1, 0])]; + tensor var_536_end_0 = const()[name = tensor("op_536_end_0"), val = tensor([1, 16, 256])]; + tensor var_536_end_mask_0 = const()[name = tensor("op_536_end_mask_0"), val = tensor([true, true, true])]; + tensor var_536 = slice_by_index(begin = var_536_begin_0, end = var_536_end_0, end_mask = var_536_end_mask_0, x = window_11)[name = tensor("op_536")]; tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; - tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_697, var_694))[name = tensor("rolled_9")]; + tensor rolled_9 = concat(axis = var_33, interleave = rolled_9_interleave_0, values = (var_536, var_533))[name = tensor("rolled_9")]; tensor clip_8 = clip(alpha = var_21, beta = var_20, x = filled_9)[name = tensor("clip_8")]; - tensor var_702 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_702")]; - tensor var_703_dtype_0 = const()[name = tensor("op_703_dtype_0"), val = tensor("fp32")]; - tensor var_704 = const()[name = tensor("op_704"), val = tensor([1, 16, 1])]; - tensor var_703 = cast(dtype = var_703_dtype_0, x = var_702)[name = tensor("cast_167")]; - tensor fill_mask_9 = reshape(shape = var_704, x = var_703)[name = tensor("fill_mask_9")]; + tensor var_541 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_541")]; + tensor var_542_dtype_0 = const()[name = tensor("op_542_dtype_0"), val = tensor("fp32")]; + tensor var_543 = const()[name = tensor("op_543"), val = tensor([1, 16, 1])]; + tensor var_542 = cast(dtype = var_542_dtype_0, x = var_541)[name = tensor("cast_167")]; + tensor fill_mask_9 = reshape(shape = var_543, x = var_542)[name = tensor("fill_mask_9")]; tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_694)[name = tensor("x_expanded_9")]; - tensor var_708 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_708")]; - tensor var_709 = mul(x = window_3, y = var_708)[name = tensor("op_709")]; - tensor var_710 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_710")]; - tensor filled_window_9 = add(x = var_709, y = var_710)[name = tensor("filled_window_9")]; - tensor var_712 = greater_equal(x = filled_9, y = var_16)[name = tensor("op_712")]; - tensor var_713_dtype_0 = const()[name = tensor("op_713_dtype_0"), val = tensor("fp32")]; - tensor var_714 = const()[name = tensor("op_714"), val = tensor([1, 1, 1])]; - tensor var_713 = cast(dtype = var_713_dtype_0, x = var_712)[name = tensor("cast_166")]; - tensor use_roll_9 = reshape(shape = var_714, x = var_713)[name = tensor("use_roll_9")]; - tensor var_716 = sub(x = var_17, y = use_roll_9)[name = tensor("op_716")]; - tensor var_717 = mul(x = filled_window_9, y = var_716)[name = tensor("op_717")]; - tensor var_718 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_718")]; - tensor input_91 = add(x = var_717, y = var_718)[name = tensor("input_91")]; - tensor var_720 = const()[name = tensor("op_720"), val = tensor(0x1p+0)]; - tensor var_721 = add(x = filled_9, y = var_720)[name = tensor("op_721")]; - tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_721)[name = tensor("clip_9")]; - tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; - tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_91)[name = tensor("x_17")]; - tensor input_93_perm_0 = const()[name = tensor("input_93_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_31_pad_type_0 = const()[name = tensor("inputs_31_pad_type_0"), val = tensor("valid")]; - tensor inputs_31_strides_0 = const()[name = tensor("inputs_31_strides_0"), val = tensor([1])]; - tensor inputs_31_pad_0 = const()[name = tensor("inputs_31_pad_0"), val = tensor([0, 0])]; - tensor inputs_31_dilations_0 = const()[name = tensor("inputs_31_dilations_0"), val = tensor([1])]; - tensor inputs_31_groups_0 = const()[name = tensor("inputs_31_groups_0"), val = tensor(1)]; - tensor input_93 = transpose(perm = input_93_perm_0, x = x_17)[name = tensor("transpose_62")]; - tensor inputs_31 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_31_dilations_0, groups = inputs_31_groups_0, pad = inputs_31_pad_0, pad_type = inputs_31_pad_type_0, strides = inputs_31_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_93)[name = tensor("inputs_31")]; - tensor var_742_split_sizes_0 = const()[name = tensor("op_742_split_sizes_0"), val = tensor([256, 256])]; - tensor var_742_axis_0 = const()[name = tensor("op_742_axis_0"), val = tensor(1)]; - tensor var_742_0, tensor var_742_1 = split(axis = var_742_axis_0, split_sizes = var_742_split_sizes_0, x = inputs_31)[name = tensor("op_742")]; - tensor var_744 = sigmoid(x = var_742_1)[name = tensor("op_744")]; - tensor inputs_33 = mul(x = var_742_0, y = var_744)[name = tensor("inputs_33")]; - tensor outputs_aug_9_pad_type_0 = const()[name = tensor("outputs_aug_9_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_9_pad_0 = const()[name = tensor("outputs_aug_9_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_9_groups_0 = const()[name = tensor("outputs_aug_9_groups_0"), val = tensor(256)]; - tensor outputs_aug_9_strides_0 = const()[name = tensor("outputs_aug_9_strides_0"), val = tensor([1])]; - tensor outputs_aug_9_dilations_0 = const()[name = tensor("outputs_aug_9_dilations_0"), val = tensor([1])]; - tensor outputs_aug_9 = conv(dilations = outputs_aug_9_dilations_0, groups = outputs_aug_9_groups_0, pad = outputs_aug_9_pad_0, pad_type = outputs_aug_9_pad_type_0, strides = outputs_aug_9_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_33)[name = tensor("outputs_aug_9")]; - tensor input_95_begin_0 = const()[name = tensor("input_95_begin_0"), val = tensor([0, 0, 0])]; - tensor input_95_end_0 = const()[name = tensor("input_95_end_0"), val = tensor([1, 256, 16])]; - tensor input_95_end_mask_0 = const()[name = tensor("input_95_end_mask_0"), val = tensor([true, true, false])]; - tensor input_95 = slice_by_index(begin = input_95_begin_0, end = input_95_end_0, end_mask = input_95_end_mask_0, x = outputs_aug_9)[name = tensor("input_95")]; - tensor inputs_35 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_95)[name = tensor("inputs_35")]; - tensor input_97 = silu(x = inputs_35)[name = tensor("input_97")]; - tensor input_99_pad_type_0 = const()[name = tensor("input_99_pad_type_0"), val = tensor("valid")]; - tensor input_99_strides_0 = const()[name = tensor("input_99_strides_0"), val = tensor([1])]; - tensor input_99_pad_0 = const()[name = tensor("input_99_pad_0"), val = tensor([0, 0])]; - tensor input_99_dilations_0 = const()[name = tensor("input_99_dilations_0"), val = tensor([1])]; - tensor input_99_groups_0 = const()[name = tensor("input_99_groups_0"), val = tensor(1)]; - tensor input_99 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_99_dilations_0, groups = input_99_groups_0, pad = input_99_pad_0, pad_type = input_99_pad_type_0, strides = input_99_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_97)[name = tensor("input_99")]; - tensor conv_out_9_perm_0 = const()[name = tensor("conv_out_9_perm_0"), val = tensor([0, 2, 1])]; - tensor var_775_begin_0 = const()[name = tensor("op_775_begin_0"), val = tensor([0, -1, 0])]; - tensor var_775_end_0 = const()[name = tensor("op_775_end_0"), val = tensor([1, 16, 256])]; - tensor var_775_end_mask_0 = const()[name = tensor("op_775_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_9 = transpose(perm = conv_out_9_perm_0, x = input_99)[name = tensor("transpose_61")]; - tensor var_775 = slice_by_index(begin = var_775_begin_0, end = var_775_end_0, end_mask = var_775_end_mask_0, x = conv_out_9)[name = tensor("op_775")]; - tensor var_778_begin_0 = const()[name = tensor("op_778_begin_0"), val = tensor([0, 1, 0])]; - tensor var_778_end_0 = const()[name = tensor("op_778_end_0"), val = tensor([1, 2, 256])]; - tensor var_778_end_mask_0 = const()[name = tensor("op_778_end_mask_0"), val = tensor([true, false, true])]; - tensor var_778 = slice_by_index(begin = var_778_begin_0, end = var_778_end_0, end_mask = var_778_end_mask_0, x = x_15)[name = tensor("op_778")]; - tensor var_781_begin_0 = const()[name = tensor("op_781_begin_0"), val = tensor([0, 1, 0])]; - tensor var_781_end_0 = const()[name = tensor("op_781_end_0"), val = tensor([1, 16, 256])]; - tensor var_781_end_mask_0 = const()[name = tensor("op_781_end_mask_0"), val = tensor([true, true, true])]; - tensor var_781 = slice_by_index(begin = var_781_begin_0, end = var_781_end_0, end_mask = var_781_end_mask_0, x = input_91)[name = tensor("op_781")]; + tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_533)[name = tensor("x_expanded_9")]; + tensor var_547 = sub(x = var_17, y = fill_mask_9)[name = tensor("op_547")]; + tensor var_548 = mul(x = window_11, y = var_547)[name = tensor("op_548")]; + tensor var_549 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_549")]; + tensor filled_window_9 = add(x = var_548, y = var_549)[name = tensor("filled_window_9")]; + tensor var_551 = greater_equal(x = filled_9, y = var_16)[name = tensor("op_551")]; + tensor var_552_dtype_0 = const()[name = tensor("op_552_dtype_0"), val = tensor("fp32")]; + tensor var_553 = const()[name = tensor("op_553"), val = tensor([1, 1, 1])]; + tensor var_552 = cast(dtype = var_552_dtype_0, x = var_551)[name = tensor("cast_166")]; + tensor use_roll_9 = reshape(shape = var_553, x = var_552)[name = tensor("use_roll_9")]; + tensor var_555 = sub(x = var_17, y = use_roll_9)[name = tensor("op_555")]; + tensor var_556 = mul(x = filled_window_9, y = var_555)[name = tensor("op_556")]; + tensor var_557 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_557")]; + tensor window_13 = add(x = var_556, y = var_557)[name = tensor("window_13")]; + tensor var_559 = const()[name = tensor("op_559"), val = tensor(0x1p+0)]; + tensor var_560 = add(x = filled_9, y = var_559)[name = tensor("op_560")]; + tensor clip_9 = clip(alpha = var_21, beta = var_16, x = var_560)[name = tensor("clip_9")]; + tensor var_563_begin_0 = const()[name = tensor("op_563_begin_0"), val = tensor([0, 1, 0])]; + tensor var_563_end_0 = const()[name = tensor("op_563_end_0"), val = tensor([1, 2, 256])]; + tensor var_563_end_mask_0 = const()[name = tensor("op_563_end_mask_0"), val = tensor([true, false, true])]; + tensor var_563 = slice_by_index(begin = var_563_begin_0, end = var_563_end_0, end_mask = var_563_end_mask_0, x = x_9)[name = tensor("op_563")]; + tensor var_566_begin_0 = const()[name = tensor("op_566_begin_0"), val = tensor([0, 1, 0])]; + tensor var_566_end_0 = const()[name = tensor("op_566_end_0"), val = tensor([1, 16, 256])]; + tensor var_566_end_mask_0 = const()[name = tensor("op_566_end_mask_0"), val = tensor([true, true, true])]; + tensor var_566 = slice_by_index(begin = var_566_begin_0, end = var_566_end_0, end_mask = var_566_end_mask_0, x = window_13)[name = tensor("op_566")]; tensor rolled_11_interleave_0 = const()[name = tensor("rolled_11_interleave_0"), val = tensor(false)]; - tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_781, var_778))[name = tensor("rolled_11")]; + tensor rolled_11 = concat(axis = var_33, interleave = rolled_11_interleave_0, values = (var_566, var_563))[name = tensor("rolled_11")]; tensor clip_10 = clip(alpha = var_21, beta = var_20, x = clip_9)[name = tensor("clip_10")]; - tensor var_786 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_786")]; - tensor var_787_dtype_0 = const()[name = tensor("op_787_dtype_0"), val = tensor("fp32")]; - tensor var_788 = const()[name = tensor("op_788"), val = tensor([1, 16, 1])]; - tensor var_787 = cast(dtype = var_787_dtype_0, x = var_786)[name = tensor("cast_165")]; - tensor fill_mask_11 = reshape(shape = var_788, x = var_787)[name = tensor("fill_mask_11")]; + tensor var_571 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_571")]; + tensor var_572_dtype_0 = const()[name = tensor("op_572_dtype_0"), val = tensor("fp32")]; + tensor var_573 = const()[name = tensor("op_573"), val = tensor([1, 16, 1])]; + tensor var_572 = cast(dtype = var_572_dtype_0, x = var_571)[name = tensor("cast_165")]; + tensor fill_mask_11 = reshape(shape = var_573, x = var_572)[name = tensor("fill_mask_11")]; tensor x_expanded_11_reps_0 = const()[name = tensor("x_expanded_11_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_778)[name = tensor("x_expanded_11")]; - tensor var_792 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_792")]; - tensor var_793 = mul(x = input_91, y = var_792)[name = tensor("op_793")]; - tensor var_794 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_794")]; - tensor filled_window_11 = add(x = var_793, y = var_794)[name = tensor("filled_window_11")]; - tensor var_796 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_796")]; - tensor var_797_dtype_0 = const()[name = tensor("op_797_dtype_0"), val = tensor("fp32")]; - tensor var_798 = const()[name = tensor("op_798"), val = tensor([1, 1, 1])]; - tensor var_797 = cast(dtype = var_797_dtype_0, x = var_796)[name = tensor("cast_164")]; - tensor use_roll_11 = reshape(shape = var_798, x = var_797)[name = tensor("use_roll_11")]; - tensor var_800 = sub(x = var_17, y = use_roll_11)[name = tensor("op_800")]; - tensor var_801 = mul(x = filled_window_11, y = var_800)[name = tensor("op_801")]; - tensor var_802 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_802")]; - tensor input_101 = add(x = var_801, y = var_802)[name = tensor("input_101")]; - tensor var_804 = const()[name = tensor("op_804"), val = tensor(0x1p+0)]; - tensor var_805 = add(x = clip_9, y = var_804)[name = tensor("op_805")]; - tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_805)[name = tensor("clip_11")]; - tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; - tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_101)[name = tensor("x_19")]; - tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_37_pad_type_0 = const()[name = tensor("inputs_37_pad_type_0"), val = tensor("valid")]; - tensor inputs_37_strides_0 = const()[name = tensor("inputs_37_strides_0"), val = tensor([1])]; - tensor inputs_37_pad_0 = const()[name = tensor("inputs_37_pad_0"), val = tensor([0, 0])]; - tensor inputs_37_dilations_0 = const()[name = tensor("inputs_37_dilations_0"), val = tensor([1])]; - tensor inputs_37_groups_0 = const()[name = tensor("inputs_37_groups_0"), val = tensor(1)]; - tensor input_103 = transpose(perm = input_103_perm_0, x = x_19)[name = tensor("transpose_60")]; - tensor inputs_37 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_37_dilations_0, groups = inputs_37_groups_0, pad = inputs_37_pad_0, pad_type = inputs_37_pad_type_0, strides = inputs_37_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_37")]; - tensor var_826_split_sizes_0 = const()[name = tensor("op_826_split_sizes_0"), val = tensor([256, 256])]; - tensor var_826_axis_0 = const()[name = tensor("op_826_axis_0"), val = tensor(1)]; - tensor var_826_0, tensor var_826_1 = split(axis = var_826_axis_0, split_sizes = var_826_split_sizes_0, x = inputs_37)[name = tensor("op_826")]; - tensor var_828 = sigmoid(x = var_826_1)[name = tensor("op_828")]; - tensor inputs_39 = mul(x = var_826_0, y = var_828)[name = tensor("inputs_39")]; - tensor outputs_aug_11_pad_type_0 = const()[name = tensor("outputs_aug_11_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_11_pad_0 = const()[name = tensor("outputs_aug_11_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_11_groups_0 = const()[name = tensor("outputs_aug_11_groups_0"), val = tensor(256)]; - tensor outputs_aug_11_strides_0 = const()[name = tensor("outputs_aug_11_strides_0"), val = tensor([1])]; - tensor outputs_aug_11_dilations_0 = const()[name = tensor("outputs_aug_11_dilations_0"), val = tensor([1])]; - tensor outputs_aug_11 = conv(dilations = outputs_aug_11_dilations_0, groups = outputs_aug_11_groups_0, pad = outputs_aug_11_pad_0, pad_type = outputs_aug_11_pad_type_0, strides = outputs_aug_11_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_39)[name = tensor("outputs_aug_11")]; - tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; - tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([1, 256, 16])]; - tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; - tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_11)[name = tensor("input_105")]; - tensor inputs_41 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_105)[name = tensor("inputs_41")]; - tensor input_107 = silu(x = inputs_41)[name = tensor("input_107")]; - tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; - tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; - tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; - tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; - tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; - tensor input_109 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; - tensor conv_out_11_perm_0 = const()[name = tensor("conv_out_11_perm_0"), val = tensor([0, 2, 1])]; - tensor var_859_begin_0 = const()[name = tensor("op_859_begin_0"), val = tensor([0, -1, 0])]; - tensor var_859_end_0 = const()[name = tensor("op_859_end_0"), val = tensor([1, 16, 256])]; - tensor var_859_end_mask_0 = const()[name = tensor("op_859_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_11 = transpose(perm = conv_out_11_perm_0, x = input_109)[name = tensor("transpose_59")]; - tensor var_859 = slice_by_index(begin = var_859_begin_0, end = var_859_end_0, end_mask = var_859_end_mask_0, x = conv_out_11)[name = tensor("op_859")]; - tensor var_862_begin_0 = const()[name = tensor("op_862_begin_0"), val = tensor([0, 2, 0])]; - tensor var_862_end_0 = const()[name = tensor("op_862_end_0"), val = tensor([1, 3, 256])]; - tensor var_862_end_mask_0 = const()[name = tensor("op_862_end_mask_0"), val = tensor([true, false, true])]; - tensor var_862 = slice_by_index(begin = var_862_begin_0, end = var_862_end_0, end_mask = var_862_end_mask_0, x = x_15)[name = tensor("op_862")]; - tensor var_865_begin_0 = const()[name = tensor("op_865_begin_0"), val = tensor([0, 1, 0])]; - tensor var_865_end_0 = const()[name = tensor("op_865_end_0"), val = tensor([1, 16, 256])]; - tensor var_865_end_mask_0 = const()[name = tensor("op_865_end_mask_0"), val = tensor([true, true, true])]; - tensor var_865 = slice_by_index(begin = var_865_begin_0, end = var_865_end_0, end_mask = var_865_end_mask_0, x = input_101)[name = tensor("op_865")]; + tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_563)[name = tensor("x_expanded_11")]; + tensor var_577 = sub(x = var_17, y = fill_mask_11)[name = tensor("op_577")]; + tensor var_578 = mul(x = window_13, y = var_577)[name = tensor("op_578")]; + tensor var_579 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_579")]; + tensor filled_window_11 = add(x = var_578, y = var_579)[name = tensor("filled_window_11")]; + tensor var_581 = greater_equal(x = clip_9, y = var_16)[name = tensor("op_581")]; + tensor var_582_dtype_0 = const()[name = tensor("op_582_dtype_0"), val = tensor("fp32")]; + tensor var_583 = const()[name = tensor("op_583"), val = tensor([1, 1, 1])]; + tensor var_582 = cast(dtype = var_582_dtype_0, x = var_581)[name = tensor("cast_164")]; + tensor use_roll_11 = reshape(shape = var_583, x = var_582)[name = tensor("use_roll_11")]; + tensor var_585 = sub(x = var_17, y = use_roll_11)[name = tensor("op_585")]; + tensor var_586 = mul(x = filled_window_11, y = var_585)[name = tensor("op_586")]; + tensor var_587 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_587")]; + tensor window_15 = add(x = var_586, y = var_587)[name = tensor("window_15")]; + tensor var_589 = const()[name = tensor("op_589"), val = tensor(0x1p+0)]; + tensor var_590 = add(x = clip_9, y = var_589)[name = tensor("op_590")]; + tensor clip_11 = clip(alpha = var_21, beta = var_16, x = var_590)[name = tensor("clip_11")]; + tensor var_593_begin_0 = const()[name = tensor("op_593_begin_0"), val = tensor([0, 2, 0])]; + tensor var_593_end_0 = const()[name = tensor("op_593_end_0"), val = tensor([1, 3, 256])]; + tensor var_593_end_mask_0 = const()[name = tensor("op_593_end_mask_0"), val = tensor([true, false, true])]; + tensor var_593 = slice_by_index(begin = var_593_begin_0, end = var_593_end_0, end_mask = var_593_end_mask_0, x = x_9)[name = tensor("op_593")]; + tensor var_596_begin_0 = const()[name = tensor("op_596_begin_0"), val = tensor([0, 1, 0])]; + tensor var_596_end_0 = const()[name = tensor("op_596_end_0"), val = tensor([1, 16, 256])]; + tensor var_596_end_mask_0 = const()[name = tensor("op_596_end_mask_0"), val = tensor([true, true, true])]; + tensor var_596 = slice_by_index(begin = var_596_begin_0, end = var_596_end_0, end_mask = var_596_end_mask_0, x = window_15)[name = tensor("op_596")]; tensor rolled_13_interleave_0 = const()[name = tensor("rolled_13_interleave_0"), val = tensor(false)]; - tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_865, var_862))[name = tensor("rolled_13")]; + tensor rolled_13 = concat(axis = var_33, interleave = rolled_13_interleave_0, values = (var_596, var_593))[name = tensor("rolled_13")]; tensor clip_12 = clip(alpha = var_21, beta = var_20, x = clip_11)[name = tensor("clip_12")]; - tensor var_870 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_870")]; - tensor var_871_dtype_0 = const()[name = tensor("op_871_dtype_0"), val = tensor("fp32")]; - tensor var_872 = const()[name = tensor("op_872"), val = tensor([1, 16, 1])]; - tensor var_871 = cast(dtype = var_871_dtype_0, x = var_870)[name = tensor("cast_163")]; - tensor fill_mask_13 = reshape(shape = var_872, x = var_871)[name = tensor("fill_mask_13")]; + tensor var_601 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_601")]; + tensor var_602_dtype_0 = const()[name = tensor("op_602_dtype_0"), val = tensor("fp32")]; + tensor var_603 = const()[name = tensor("op_603"), val = tensor([1, 16, 1])]; + tensor var_602 = cast(dtype = var_602_dtype_0, x = var_601)[name = tensor("cast_163")]; + tensor fill_mask_13 = reshape(shape = var_603, x = var_602)[name = tensor("fill_mask_13")]; tensor x_expanded_13_reps_0 = const()[name = tensor("x_expanded_13_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_862)[name = tensor("x_expanded_13")]; - tensor var_876 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_876")]; - tensor var_877 = mul(x = input_101, y = var_876)[name = tensor("op_877")]; - tensor var_878 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_878")]; - tensor filled_window_13 = add(x = var_877, y = var_878)[name = tensor("filled_window_13")]; - tensor var_880 = greater_equal(x = clip_11, y = var_16)[name = tensor("op_880")]; - tensor var_881_dtype_0 = const()[name = tensor("op_881_dtype_0"), val = tensor("fp32")]; - tensor var_882 = const()[name = tensor("op_882"), val = tensor([1, 1, 1])]; - tensor var_881 = cast(dtype = var_881_dtype_0, x = var_880)[name = tensor("cast_162")]; - tensor use_roll_13 = reshape(shape = var_882, x = var_881)[name = tensor("use_roll_13")]; - tensor var_884 = sub(x = var_17, y = use_roll_13)[name = tensor("op_884")]; - tensor var_885 = mul(x = filled_window_13, y = var_884)[name = tensor("op_885")]; - tensor var_886 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_886")]; - tensor input_111 = add(x = var_885, y = var_886)[name = tensor("input_111")]; - tensor var_888 = const()[name = tensor("op_888"), val = tensor(0x1p+0)]; - tensor var_889 = add(x = clip_11, y = var_888)[name = tensor("op_889")]; - tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_889)[name = tensor("clip_13")]; - tensor x_21_axes_0 = const()[name = tensor("x_21_axes_0"), val = tensor([-1])]; - tensor x_21 = layer_norm(axes = x_21_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_111)[name = tensor("x_21")]; - tensor input_113_perm_0 = const()[name = tensor("input_113_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_43_pad_type_0 = const()[name = tensor("inputs_43_pad_type_0"), val = tensor("valid")]; - tensor inputs_43_strides_0 = const()[name = tensor("inputs_43_strides_0"), val = tensor([1])]; - tensor inputs_43_pad_0 = const()[name = tensor("inputs_43_pad_0"), val = tensor([0, 0])]; - tensor inputs_43_dilations_0 = const()[name = tensor("inputs_43_dilations_0"), val = tensor([1])]; - tensor inputs_43_groups_0 = const()[name = tensor("inputs_43_groups_0"), val = tensor(1)]; - tensor input_113 = transpose(perm = input_113_perm_0, x = x_21)[name = tensor("transpose_58")]; - tensor inputs_43 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_43_dilations_0, groups = inputs_43_groups_0, pad = inputs_43_pad_0, pad_type = inputs_43_pad_type_0, strides = inputs_43_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_113)[name = tensor("inputs_43")]; - tensor var_910_split_sizes_0 = const()[name = tensor("op_910_split_sizes_0"), val = tensor([256, 256])]; - tensor var_910_axis_0 = const()[name = tensor("op_910_axis_0"), val = tensor(1)]; - tensor var_910_0, tensor var_910_1 = split(axis = var_910_axis_0, split_sizes = var_910_split_sizes_0, x = inputs_43)[name = tensor("op_910")]; - tensor var_912 = sigmoid(x = var_910_1)[name = tensor("op_912")]; - tensor inputs_45 = mul(x = var_910_0, y = var_912)[name = tensor("inputs_45")]; - tensor outputs_aug_13_pad_type_0 = const()[name = tensor("outputs_aug_13_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_13_pad_0 = const()[name = tensor("outputs_aug_13_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_13_groups_0 = const()[name = tensor("outputs_aug_13_groups_0"), val = tensor(256)]; - tensor outputs_aug_13_strides_0 = const()[name = tensor("outputs_aug_13_strides_0"), val = tensor([1])]; - tensor outputs_aug_13_dilations_0 = const()[name = tensor("outputs_aug_13_dilations_0"), val = tensor([1])]; - tensor outputs_aug_13 = conv(dilations = outputs_aug_13_dilations_0, groups = outputs_aug_13_groups_0, pad = outputs_aug_13_pad_0, pad_type = outputs_aug_13_pad_type_0, strides = outputs_aug_13_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_45)[name = tensor("outputs_aug_13")]; - tensor input_115_begin_0 = const()[name = tensor("input_115_begin_0"), val = tensor([0, 0, 0])]; - tensor input_115_end_0 = const()[name = tensor("input_115_end_0"), val = tensor([1, 256, 16])]; - tensor input_115_end_mask_0 = const()[name = tensor("input_115_end_mask_0"), val = tensor([true, true, false])]; - tensor input_115 = slice_by_index(begin = input_115_begin_0, end = input_115_end_0, end_mask = input_115_end_mask_0, x = outputs_aug_13)[name = tensor("input_115")]; - tensor inputs_47 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_115)[name = tensor("inputs_47")]; - tensor input_117 = silu(x = inputs_47)[name = tensor("input_117")]; - tensor input_119_pad_type_0 = const()[name = tensor("input_119_pad_type_0"), val = tensor("valid")]; - tensor input_119_strides_0 = const()[name = tensor("input_119_strides_0"), val = tensor([1])]; - tensor input_119_pad_0 = const()[name = tensor("input_119_pad_0"), val = tensor([0, 0])]; - tensor input_119_dilations_0 = const()[name = tensor("input_119_dilations_0"), val = tensor([1])]; - tensor input_119_groups_0 = const()[name = tensor("input_119_groups_0"), val = tensor(1)]; - tensor input_119 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_119_dilations_0, groups = input_119_groups_0, pad = input_119_pad_0, pad_type = input_119_pad_type_0, strides = input_119_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_117)[name = tensor("input_119")]; - tensor conv_out_13_perm_0 = const()[name = tensor("conv_out_13_perm_0"), val = tensor([0, 2, 1])]; - tensor var_943_begin_0 = const()[name = tensor("op_943_begin_0"), val = tensor([0, -1, 0])]; - tensor var_943_end_0 = const()[name = tensor("op_943_end_0"), val = tensor([1, 16, 256])]; - tensor var_943_end_mask_0 = const()[name = tensor("op_943_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_13 = transpose(perm = conv_out_13_perm_0, x = input_119)[name = tensor("transpose_57")]; - tensor var_943 = slice_by_index(begin = var_943_begin_0, end = var_943_end_0, end_mask = var_943_end_mask_0, x = conv_out_13)[name = tensor("op_943")]; - tensor var_946_begin_0 = const()[name = tensor("op_946_begin_0"), val = tensor([0, 3, 0])]; - tensor var_946_end_0 = const()[name = tensor("op_946_end_0"), val = tensor([1, 1, 256])]; - tensor var_946_end_mask_0 = const()[name = tensor("op_946_end_mask_0"), val = tensor([true, true, true])]; - tensor var_946 = slice_by_index(begin = var_946_begin_0, end = var_946_end_0, end_mask = var_946_end_mask_0, x = x_15)[name = tensor("op_946")]; - tensor var_949_begin_0 = const()[name = tensor("op_949_begin_0"), val = tensor([0, 1, 0])]; - tensor var_949_end_0 = const()[name = tensor("op_949_end_0"), val = tensor([1, 16, 256])]; - tensor var_949_end_mask_0 = const()[name = tensor("op_949_end_mask_0"), val = tensor([true, true, true])]; - tensor var_949 = slice_by_index(begin = var_949_begin_0, end = var_949_end_0, end_mask = var_949_end_mask_0, x = input_111)[name = tensor("op_949")]; + tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_593)[name = tensor("x_expanded_13")]; + tensor var_607 = sub(x = var_17, y = fill_mask_13)[name = tensor("op_607")]; + tensor var_608 = mul(x = window_15, y = var_607)[name = tensor("op_608")]; + tensor var_609 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_609")]; + tensor filled_window_13 = add(x = var_608, y = var_609)[name = tensor("filled_window_13")]; + tensor var_611 = greater_equal(x = clip_11, y = var_16)[name = tensor("op_611")]; + tensor var_612_dtype_0 = const()[name = tensor("op_612_dtype_0"), val = tensor("fp32")]; + tensor var_613 = const()[name = tensor("op_613"), val = tensor([1, 1, 1])]; + tensor var_612 = cast(dtype = var_612_dtype_0, x = var_611)[name = tensor("cast_162")]; + tensor use_roll_13 = reshape(shape = var_613, x = var_612)[name = tensor("use_roll_13")]; + tensor var_615 = sub(x = var_17, y = use_roll_13)[name = tensor("op_615")]; + tensor var_616 = mul(x = filled_window_13, y = var_615)[name = tensor("op_616")]; + tensor var_617 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_617")]; + tensor window_17 = add(x = var_616, y = var_617)[name = tensor("window_17")]; + tensor var_619 = const()[name = tensor("op_619"), val = tensor(0x1p+0)]; + tensor var_620 = add(x = clip_11, y = var_619)[name = tensor("op_620")]; + tensor clip_13 = clip(alpha = var_21, beta = var_16, x = var_620)[name = tensor("clip_13")]; + tensor var_623_begin_0 = const()[name = tensor("op_623_begin_0"), val = tensor([0, 3, 0])]; + tensor var_623_end_0 = const()[name = tensor("op_623_end_0"), val = tensor([1, 1, 256])]; + tensor var_623_end_mask_0 = const()[name = tensor("op_623_end_mask_0"), val = tensor([true, true, true])]; + tensor var_623 = slice_by_index(begin = var_623_begin_0, end = var_623_end_0, end_mask = var_623_end_mask_0, x = x_9)[name = tensor("op_623")]; + tensor var_626_begin_0 = const()[name = tensor("op_626_begin_0"), val = tensor([0, 1, 0])]; + tensor var_626_end_0 = const()[name = tensor("op_626_end_0"), val = tensor([1, 16, 256])]; + tensor var_626_end_mask_0 = const()[name = tensor("op_626_end_mask_0"), val = tensor([true, true, true])]; + tensor var_626 = slice_by_index(begin = var_626_begin_0, end = var_626_end_0, end_mask = var_626_end_mask_0, x = window_17)[name = tensor("op_626")]; tensor rolled_15_interleave_0 = const()[name = tensor("rolled_15_interleave_0"), val = tensor(false)]; - tensor rolled_15 = concat(axis = var_33, interleave = rolled_15_interleave_0, values = (var_949, var_946))[name = tensor("rolled_15")]; + tensor rolled_15 = concat(axis = var_33, interleave = rolled_15_interleave_0, values = (var_626, var_623))[name = tensor("rolled_15")]; tensor clip_14 = clip(alpha = var_21, beta = var_20, x = clip_13)[name = tensor("clip_14")]; - tensor var_954 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_954")]; - tensor var_955_dtype_0 = const()[name = tensor("op_955_dtype_0"), val = tensor("fp32")]; - tensor var_956 = const()[name = tensor("op_956"), val = tensor([1, 16, 1])]; - tensor var_955 = cast(dtype = var_955_dtype_0, x = var_954)[name = tensor("cast_161")]; - tensor fill_mask_15 = reshape(shape = var_956, x = var_955)[name = tensor("fill_mask_15")]; + tensor var_631 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_631")]; + tensor var_632_dtype_0 = const()[name = tensor("op_632_dtype_0"), val = tensor("fp32")]; + tensor var_633 = const()[name = tensor("op_633"), val = tensor([1, 16, 1])]; + tensor var_632 = cast(dtype = var_632_dtype_0, x = var_631)[name = tensor("cast_161")]; + tensor fill_mask_15 = reshape(shape = var_633, x = var_632)[name = tensor("fill_mask_15")]; tensor x_expanded_15_reps_0 = const()[name = tensor("x_expanded_15_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_946)[name = tensor("x_expanded_15")]; - tensor var_960 = sub(x = var_17, y = fill_mask_15)[name = tensor("op_960")]; - tensor var_961 = mul(x = input_111, y = var_960)[name = tensor("op_961")]; - tensor var_962 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_962")]; - tensor filled_window_15 = add(x = var_961, y = var_962)[name = tensor("filled_window_15")]; - tensor var_964 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_964")]; - tensor var_965_dtype_0 = const()[name = tensor("op_965_dtype_0"), val = tensor("fp32")]; - tensor var_966 = const()[name = tensor("op_966"), val = tensor([1, 1, 1])]; - tensor var_965 = cast(dtype = var_965_dtype_0, x = var_964)[name = tensor("cast_160")]; - tensor use_roll_15 = reshape(shape = var_966, x = var_965)[name = tensor("use_roll_15")]; - tensor var_968 = sub(x = var_17, y = use_roll_15)[name = tensor("op_968")]; - tensor var_969 = mul(x = filled_window_15, y = var_968)[name = tensor("op_969")]; - tensor var_970 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_970")]; - tensor input_121 = add(x = var_969, y = var_970)[name = tensor("input_121")]; - tensor var_972 = const()[name = tensor("op_972"), val = tensor(0x1p+0)]; - tensor var_973 = add(x = clip_13, y = var_972)[name = tensor("op_973")]; - tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_973)[name = tensor("clip_15")]; - tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; - tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_121)[name = tensor("x_23")]; - tensor input_123_perm_0 = const()[name = tensor("input_123_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_49_pad_type_0 = const()[name = tensor("inputs_49_pad_type_0"), val = tensor("valid")]; - tensor inputs_49_strides_0 = const()[name = tensor("inputs_49_strides_0"), val = tensor([1])]; - tensor inputs_49_pad_0 = const()[name = tensor("inputs_49_pad_0"), val = tensor([0, 0])]; - tensor inputs_49_dilations_0 = const()[name = tensor("inputs_49_dilations_0"), val = tensor([1])]; - tensor inputs_49_groups_0 = const()[name = tensor("inputs_49_groups_0"), val = tensor(1)]; - tensor input_123 = transpose(perm = input_123_perm_0, x = x_23)[name = tensor("transpose_56")]; - tensor inputs_49 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_49_dilations_0, groups = inputs_49_groups_0, pad = inputs_49_pad_0, pad_type = inputs_49_pad_type_0, strides = inputs_49_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_123)[name = tensor("inputs_49")]; - tensor var_994_split_sizes_0 = const()[name = tensor("op_994_split_sizes_0"), val = tensor([256, 256])]; - tensor var_994_axis_0 = const()[name = tensor("op_994_axis_0"), val = tensor(1)]; - tensor var_994_0, tensor var_994_1 = split(axis = var_994_axis_0, split_sizes = var_994_split_sizes_0, x = inputs_49)[name = tensor("op_994")]; - tensor var_996 = sigmoid(x = var_994_1)[name = tensor("op_996")]; - tensor inputs_51 = mul(x = var_994_0, y = var_996)[name = tensor("inputs_51")]; - tensor outputs_aug_15_pad_type_0 = const()[name = tensor("outputs_aug_15_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_15_pad_0 = const()[name = tensor("outputs_aug_15_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_15_groups_0 = const()[name = tensor("outputs_aug_15_groups_0"), val = tensor(256)]; - tensor outputs_aug_15_strides_0 = const()[name = tensor("outputs_aug_15_strides_0"), val = tensor([1])]; - tensor outputs_aug_15_dilations_0 = const()[name = tensor("outputs_aug_15_dilations_0"), val = tensor([1])]; - tensor outputs_aug_15 = conv(dilations = outputs_aug_15_dilations_0, groups = outputs_aug_15_groups_0, pad = outputs_aug_15_pad_0, pad_type = outputs_aug_15_pad_type_0, strides = outputs_aug_15_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_51)[name = tensor("outputs_aug_15")]; - tensor input_125_begin_0 = const()[name = tensor("input_125_begin_0"), val = tensor([0, 0, 0])]; - tensor input_125_end_0 = const()[name = tensor("input_125_end_0"), val = tensor([1, 256, 16])]; - tensor input_125_end_mask_0 = const()[name = tensor("input_125_end_mask_0"), val = tensor([true, true, false])]; - tensor input_125 = slice_by_index(begin = input_125_begin_0, end = input_125_end_0, end_mask = input_125_end_mask_0, x = outputs_aug_15)[name = tensor("input_125")]; - tensor inputs_53 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_125)[name = tensor("inputs_53")]; - tensor input_127 = silu(x = inputs_53)[name = tensor("input_127")]; - tensor input_129_pad_type_0 = const()[name = tensor("input_129_pad_type_0"), val = tensor("valid")]; - tensor input_129_strides_0 = const()[name = tensor("input_129_strides_0"), val = tensor([1])]; - tensor input_129_pad_0 = const()[name = tensor("input_129_pad_0"), val = tensor([0, 0])]; - tensor input_129_dilations_0 = const()[name = tensor("input_129_dilations_0"), val = tensor([1])]; - tensor input_129_groups_0 = const()[name = tensor("input_129_groups_0"), val = tensor(1)]; - tensor input_129 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_129_dilations_0, groups = input_129_groups_0, pad = input_129_pad_0, pad_type = input_129_pad_type_0, strides = input_129_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_127)[name = tensor("input_129")]; - tensor conv_out_15_perm_0 = const()[name = tensor("conv_out_15_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1027_begin_0 = const()[name = tensor("op_1027_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1027_end_0 = const()[name = tensor("op_1027_end_0"), val = tensor([1, 16, 256])]; - tensor var_1027_end_mask_0 = const()[name = tensor("op_1027_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_15 = transpose(perm = conv_out_15_perm_0, x = input_129)[name = tensor("transpose_55")]; - tensor var_1027 = slice_by_index(begin = var_1027_begin_0, end = var_1027_end_0, end_mask = var_1027_end_mask_0, x = conv_out_15)[name = tensor("op_1027")]; - tensor var_1030_interleave_0 = const()[name = tensor("op_1030_interleave_0"), val = tensor(false)]; - tensor var_1030 = concat(axis = var_33, interleave = var_1030_interleave_0, values = (var_775, var_859, var_943, var_1027))[name = tensor("op_1030")]; - tensor input_131 = add(x = x_15, y = var_1030)[name = tensor("input_131")]; - tensor input_133_axes_0 = const()[name = tensor("input_133_axes_0"), val = tensor([-1])]; - tensor input_133 = layer_norm(axes = input_133_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_131)[name = tensor("input_133")]; - tensor inputs_55 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_133)[name = tensor("linear_17")]; - tensor input_135 = silu(x = inputs_55)[name = tensor("input_135")]; - tensor input_139 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_135)[name = tensor("linear_18")]; - tensor var_1053 = const()[name = tensor("op_1053"), val = tensor(0x1p-1)]; - tensor var_1054 = mul(x = input_139, y = var_1053)[name = tensor("op_1054")]; - tensor input_141 = add(x = var_1054, y = input_131)[name = tensor("input_141")]; - tensor input_143_axes_0 = const()[name = tensor("input_143_axes_0"), val = tensor([-1])]; - tensor input_143 = layer_norm(axes = input_143_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_141)[name = tensor("input_143")]; - tensor input_145_axes_0 = const()[name = tensor("input_145_axes_0"), val = tensor([-1])]; - tensor input_145 = layer_norm(axes = input_145_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_143)[name = tensor("input_145")]; - tensor inputs_57 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_145)[name = tensor("linear_19")]; - tensor input_147 = silu(x = inputs_57)[name = tensor("input_147")]; - tensor input_151 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_147)[name = tensor("linear_20")]; - tensor var_1083 = const()[name = tensor("op_1083"), val = tensor(0x1p-1)]; - tensor var_1084 = mul(x = input_151, y = var_1083)[name = tensor("op_1084")]; - tensor input_153 = add(x = var_1084, y = input_143)[name = tensor("input_153")]; - tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; - tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_153)[name = tensor("x_25")]; + tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_623)[name = tensor("x_expanded_15")]; + tensor var_637 = sub(x = var_17, y = fill_mask_15)[name = tensor("op_637")]; + tensor var_638 = mul(x = window_17, y = var_637)[name = tensor("op_638")]; + tensor var_639 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_639")]; + tensor filled_window_15 = add(x = var_638, y = var_639)[name = tensor("filled_window_15")]; + tensor var_641 = greater_equal(x = clip_13, y = var_16)[name = tensor("op_641")]; + tensor var_642_dtype_0 = const()[name = tensor("op_642_dtype_0"), val = tensor("fp32")]; + tensor var_643 = const()[name = tensor("op_643"), val = tensor([1, 1, 1])]; + tensor var_642 = cast(dtype = var_642_dtype_0, x = var_641)[name = tensor("cast_160")]; + tensor use_roll_15 = reshape(shape = var_643, x = var_642)[name = tensor("use_roll_15")]; + tensor var_645 = sub(x = var_17, y = use_roll_15)[name = tensor("op_645")]; + tensor var_646 = mul(x = filled_window_15, y = var_645)[name = tensor("op_646")]; + tensor var_647 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_647")]; + tensor window_19 = add(x = var_646, y = var_647)[name = tensor("window_19")]; + tensor var_649 = const()[name = tensor("op_649"), val = tensor(0x1p+0)]; + tensor var_650 = add(x = clip_13, y = var_649)[name = tensor("op_650")]; + tensor clip_15 = clip(alpha = var_21, beta = var_16, x = var_650)[name = tensor("clip_15")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_30, interleave = input_61_interleave_0, values = (window_13, window_15, window_17, window_19))[name = tensor("input_61")]; + tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; + tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; + tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_13_pad_type_0 = const()[name = tensor("inputs_13_pad_type_0"), val = tensor("valid")]; + tensor inputs_13_strides_0 = const()[name = tensor("inputs_13_strides_0"), val = tensor([1])]; + tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; + tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; + tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; + tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; + tensor var_673_split_sizes_0 = const()[name = tensor("op_673_split_sizes_0"), val = tensor([256, 256])]; + tensor var_673_axis_0 = const()[name = tensor("op_673_axis_0"), val = tensor(1)]; + tensor var_673_0, tensor var_673_1 = split(axis = var_673_axis_0, split_sizes = var_673_split_sizes_0, x = inputs_13)[name = tensor("op_673")]; + tensor var_675 = sigmoid(x = var_673_1)[name = tensor("op_675")]; + tensor inputs_15 = mul(x = var_673_0, y = var_675)[name = tensor("inputs_15")]; + tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; + tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; + tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; + tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_15)[name = tensor("outputs_aug_3")]; + tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; + tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([4, 256, 16])]; + tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; + tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_3)[name = tensor("input_65")]; + tensor inputs_17 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_65)[name = tensor("inputs_17")]; + tensor input_67 = silu(x = inputs_17)[name = tensor("input_67")]; + tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; + tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; + tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; + tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; + tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; + tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; + tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; + tensor var_706_begin_0 = const()[name = tensor("op_706_begin_0"), val = tensor([0, -1, 0])]; + tensor var_706_end_0 = const()[name = tensor("op_706_end_0"), val = tensor([4, 16, 256])]; + tensor var_706_end_mask_0 = const()[name = tensor("op_706_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_706 = slice_by_index(begin = var_706_begin_0, end = var_706_end_0, end_mask = var_706_end_mask_0, x = conv_out_3)[name = tensor("op_706")]; + tensor var_708_perm_0 = const()[name = tensor("op_708_perm_0"), val = tensor([1, 0, 2])]; + tensor var_708 = transpose(perm = var_708_perm_0, x = var_706)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_708)[name = tensor("input_71")]; + tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; + tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; + tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; + tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; + tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; + tensor var_731 = const()[name = tensor("op_731"), val = tensor(0x1p-1)]; + tensor var_732 = mul(x = input_79, y = var_731)[name = tensor("op_732")]; + tensor input_81 = add(x = var_732, y = input_71)[name = tensor("input_81")]; + tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; + tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_35, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; + tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; + tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; + tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; + tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; + tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; + tensor var_761 = const()[name = tensor("op_761"), val = tensor(0x1p-1)]; + tensor var_762 = mul(x = input_91, y = var_761)[name = tensor("op_762")]; + tensor input_93 = add(x = var_762, y = input_83)[name = tensor("input_93")]; + tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; + tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_35, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; tensor prev_kv_5_end_0 = const()[name = tensor("prev_kv_5_end_0"), val = tensor([3, 1, 4, 64, 64])]; tensor prev_kv_5_end_mask_0 = const()[name = tensor("prev_kv_5_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1029,389 +800,274 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_1098 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_25)[name = tensor("linear_21")]; - tensor var_1099 = const()[name = tensor("op_1099"), val = tensor([1, 4, 4, 64])]; - tensor var_1100 = reshape(shape = var_1099, x = var_1098)[name = tensor("op_1100")]; + tensor var_776 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_777 = const()[name = tensor("op_777"), val = tensor([1, 4, 4, 64])]; + tensor var_778 = reshape(shape = var_777, x = var_776)[name = tensor("op_778")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1104 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_25)[name = tensor("linear_22")]; - tensor var_1105 = const()[name = tensor("op_1105"), val = tensor(0x1p-3)]; - tensor var_1106 = mul(x = var_1104, y = var_1105)[name = tensor("op_1106")]; - tensor var_1107 = const()[name = tensor("op_1107"), val = tensor([1, 4, 4, 64])]; - tensor var_1108 = reshape(shape = var_1107, x = var_1106)[name = tensor("op_1108")]; + tensor var_782 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_783 = const()[name = tensor("op_783"), val = tensor(0x1p-3)]; + tensor var_784 = mul(x = var_782, y = var_783)[name = tensor("op_784")]; + tensor var_785 = const()[name = tensor("op_785"), val = tensor([1, 4, 4, 64])]; + tensor var_786 = reshape(shape = var_785, x = var_784)[name = tensor("op_786")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1112 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_25)[name = tensor("linear_23")]; - tensor var_1113 = const()[name = tensor("op_1113"), val = tensor([1, 4, 4, 64])]; - tensor var_1114 = reshape(shape = var_1113, x = var_1112)[name = tensor("op_1114")]; + tensor var_790 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_791 = const()[name = tensor("op_791"), val = tensor([1, 4, 4, 64])]; + tensor var_792 = reshape(shape = var_791, x = var_790)[name = tensor("op_792")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_157 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_25)[name = tensor("linear_24")]; + tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; tensor s_t_5 = add(x = prev_scale_5, y = encoder__t_index)[name = tensor("s_t_5")]; tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_1108)[name = tensor("transpose_53")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_1100)[name = tensor("transpose_54")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_786)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_778)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_1124 = const()[name = tensor("op_1124"), val = tensor([4, 1])]; - tensor var_1125 = reshape(shape = var_1124, x = sqrt_s_t_5)[name = tensor("op_1125")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_1125)[name = tensor("M_5")]; - tensor var_1127 = mul(x = qk_5, y = M_5)[name = tensor("op_1127")]; + tensor var_802 = const()[name = tensor("op_802"), val = tensor([4, 1])]; + tensor var_803 = reshape(shape = var_802, x = sqrt_s_t_5)[name = tensor("op_803")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_803)[name = tensor("M_5")]; + tensor var_805 = mul(x = qk_5, y = M_5)[name = tensor("op_805")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_1114)[name = tensor("transpose_52")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_1127, y = v_5)[name = tensor("inner_5")]; - tensor var_1129_transpose_x_0 = const()[name = tensor("op_1129_transpose_x_0"), val = tensor(false)]; - tensor var_1129_transpose_y_0 = const()[name = tensor("op_1129_transpose_y_0"), val = tensor(false)]; - tensor var_1129 = matmul(transpose_x = var_1129_transpose_x_0, transpose_y = var_1129_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_1129")]; - tensor var_1130 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_1130")]; - tensor var_1131 = const()[name = tensor("op_1131"), val = tensor([1, 1, 4, 1])]; - tensor var_1132 = reshape(shape = var_1131, x = var_1130)[name = tensor("op_1132")]; - tensor cross_5 = mul(x = var_1129, y = var_1132)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_792)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_805, y = v_5)[name = tensor("inner_5")]; + tensor var_807_transpose_x_0 = const()[name = tensor("op_807_transpose_x_0"), val = tensor(false)]; + tensor var_807_transpose_y_0 = const()[name = tensor("op_807_transpose_y_0"), val = tensor(false)]; + tensor var_807 = matmul(transpose_x = var_807_transpose_x_0, transpose_y = var_807_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_807")]; + tensor var_808 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_808")]; + tensor var_809 = const()[name = tensor("op_809"), val = tensor([1, 1, 4, 1])]; + tensor var_810 = reshape(shape = var_809, x = var_808)[name = tensor("op_810")]; + tensor cross_5 = mul(x = var_807, y = var_810)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_1135 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_1135")]; - tensor var_1137_transpose_x_1 = const()[name = tensor("op_1137_transpose_x_1"), val = tensor(true)]; - tensor var_1137_transpose_y_1 = const()[name = tensor("op_1137_transpose_y_1"), val = tensor(false)]; - tensor var_1137 = matmul(transpose_x = var_1137_transpose_x_1, transpose_y = var_1137_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_1137")]; - tensor new_kv_unnorm_5 = add(x = var_1135, y = var_1137)[name = tensor("new_kv_unnorm_5")]; - tensor var_1139 = const()[name = tensor("op_1139"), val = tensor(0x1p+2)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_1139)[name = tensor("new_scale_5")]; - tensor var_1141 = sqrt(x = new_scale_5)[name = tensor("op_1141")]; - tensor var_1142 = real_div(x = new_kv_unnorm_5, y = var_1141)[name = tensor("op_1142")]; - tensor var_1143_perm_0 = const()[name = tensor("op_1143_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_813 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_813")]; + tensor var_815_transpose_x_1 = const()[name = tensor("op_815_transpose_x_1"), val = tensor(true)]; + tensor var_815_transpose_y_1 = const()[name = tensor("op_815_transpose_y_1"), val = tensor(false)]; + tensor var_815 = matmul(transpose_x = var_815_transpose_x_1, transpose_y = var_815_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_815")]; + tensor new_kv_unnorm_5 = add(x = var_813, y = var_815)[name = tensor("new_kv_unnorm_5")]; + tensor var_817 = const()[name = tensor("op_817"), val = tensor(0x1p+2)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_817)[name = tensor("new_scale_5")]; + tensor var_819 = sqrt(x = new_scale_5)[name = tensor("op_819")]; + tensor var_820 = real_div(x = new_kv_unnorm_5, y = var_819)[name = tensor("op_820")]; + tensor var_821_perm_0 = const()[name = tensor("op_821_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_1143 = transpose(perm = var_1143_perm_0, x = out_13)[name = tensor("transpose_51")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_1143)[name = tensor("out_15")]; - tensor var_1147 = const()[name = tensor("op_1147"), val = tensor([1, 4, 256])]; - tensor out_17 = reshape(shape = var_1147, x = out_15)[name = tensor("out_17")]; - tensor var_1149 = silu(x = input_157)[name = tensor("op_1149")]; - tensor input_159 = mul(x = var_1149, y = out_17)[name = tensor("input_159")]; - tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_159)[name = tensor("linear_25")]; - tensor x_27 = add(x = input_153, y = ret_out_5)[name = tensor("x_27")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor var_821 = transpose(perm = var_821_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_24, x = var_821)[name = tensor("out_15")]; + tensor var_825 = const()[name = tensor("op_825"), val = tensor([1, 4, 256])]; + tensor out_17 = reshape(shape = var_825, x = out_15)[name = tensor("out_17")]; + tensor var_827 = silu(x = input_97)[name = tensor("op_827")]; + tensor input_99 = mul(x = var_827, y = out_17)[name = tensor("input_99")]; + tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; + tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; + tensor window_21_begin_0 = const()[name = tensor("window_21_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_21_end_0 = const()[name = tensor("window_21_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_21_end_mask_0 = const()[name = tensor("window_21_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_21_squeeze_mask_0 = const()[name = tensor("window_21_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_21 = slice_by_index(begin = window_21_begin_0, end = window_21_end_0, end_mask = window_21_end_mask_0, squeeze_mask = window_21_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_21")]; tensor filled_17_begin_0 = const()[name = tensor("filled_17_begin_0"), val = tensor([2, 0])]; tensor filled_17_end_0 = const()[name = tensor("filled_17_end_0"), val = tensor([3, 1])]; tensor filled_17_end_mask_0 = const()[name = tensor("filled_17_end_mask_0"), val = tensor([false, true])]; tensor filled_17_squeeze_mask_0 = const()[name = tensor("filled_17_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_17 = slice_by_index(begin = filled_17_begin_0, end = filled_17_end_0, end_mask = filled_17_end_mask_0, squeeze_mask = filled_17_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_17")]; - tensor var_1158_begin_0 = const()[name = tensor("op_1158_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1158_end_0 = const()[name = tensor("op_1158_end_0"), val = tensor([1, 1, 256])]; - tensor var_1158_end_mask_0 = const()[name = tensor("op_1158_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1158 = slice_by_index(begin = var_1158_begin_0, end = var_1158_end_0, end_mask = var_1158_end_mask_0, x = x_27)[name = tensor("op_1158")]; - tensor var_1161_begin_0 = const()[name = tensor("op_1161_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1161_end_0 = const()[name = tensor("op_1161_end_0"), val = tensor([1, 16, 256])]; - tensor var_1161_end_mask_0 = const()[name = tensor("op_1161_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1161 = slice_by_index(begin = var_1161_begin_0, end = var_1161_end_0, end_mask = var_1161_end_mask_0, x = window_5)[name = tensor("op_1161")]; + tensor var_836_begin_0 = const()[name = tensor("op_836_begin_0"), val = tensor([0, 0, 0])]; + tensor var_836_end_0 = const()[name = tensor("op_836_end_0"), val = tensor([1, 1, 256])]; + tensor var_836_end_mask_0 = const()[name = tensor("op_836_end_mask_0"), val = tensor([true, false, true])]; + tensor var_836 = slice_by_index(begin = var_836_begin_0, end = var_836_end_0, end_mask = var_836_end_mask_0, x = x_15)[name = tensor("op_836")]; + tensor var_839_begin_0 = const()[name = tensor("op_839_begin_0"), val = tensor([0, 1, 0])]; + tensor var_839_end_0 = const()[name = tensor("op_839_end_0"), val = tensor([1, 16, 256])]; + tensor var_839_end_mask_0 = const()[name = tensor("op_839_end_mask_0"), val = tensor([true, true, true])]; + tensor var_839 = slice_by_index(begin = var_839_begin_0, end = var_839_end_0, end_mask = var_839_end_mask_0, x = window_21)[name = tensor("op_839")]; tensor rolled_17_interleave_0 = const()[name = tensor("rolled_17_interleave_0"), val = tensor(false)]; - tensor rolled_17 = concat(axis = var_33, interleave = rolled_17_interleave_0, values = (var_1161, var_1158))[name = tensor("rolled_17")]; + tensor rolled_17 = concat(axis = var_33, interleave = rolled_17_interleave_0, values = (var_839, var_836))[name = tensor("rolled_17")]; tensor clip_16 = clip(alpha = var_21, beta = var_20, x = filled_17)[name = tensor("clip_16")]; - tensor var_1166 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_1166")]; - tensor var_1167_dtype_0 = const()[name = tensor("op_1167_dtype_0"), val = tensor("fp32")]; - tensor var_1168 = const()[name = tensor("op_1168"), val = tensor([1, 16, 1])]; - tensor var_1167 = cast(dtype = var_1167_dtype_0, x = var_1166)[name = tensor("cast_159")]; - tensor fill_mask_17 = reshape(shape = var_1168, x = var_1167)[name = tensor("fill_mask_17")]; + tensor var_844 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_844")]; + tensor var_845_dtype_0 = const()[name = tensor("op_845_dtype_0"), val = tensor("fp32")]; + tensor var_846 = const()[name = tensor("op_846"), val = tensor([1, 16, 1])]; + tensor var_845 = cast(dtype = var_845_dtype_0, x = var_844)[name = tensor("cast_159")]; + tensor fill_mask_17 = reshape(shape = var_846, x = var_845)[name = tensor("fill_mask_17")]; tensor x_expanded_17_reps_0 = const()[name = tensor("x_expanded_17_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_1158)[name = tensor("x_expanded_17")]; - tensor var_1172 = sub(x = var_17, y = fill_mask_17)[name = tensor("op_1172")]; - tensor var_1173 = mul(x = window_5, y = var_1172)[name = tensor("op_1173")]; - tensor var_1174 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_1174")]; - tensor filled_window_17 = add(x = var_1173, y = var_1174)[name = tensor("filled_window_17")]; - tensor var_1176 = greater_equal(x = filled_17, y = var_16)[name = tensor("op_1176")]; - tensor var_1177_dtype_0 = const()[name = tensor("op_1177_dtype_0"), val = tensor("fp32")]; - tensor var_1178 = const()[name = tensor("op_1178"), val = tensor([1, 1, 1])]; - tensor var_1177 = cast(dtype = var_1177_dtype_0, x = var_1176)[name = tensor("cast_158")]; - tensor use_roll_17 = reshape(shape = var_1178, x = var_1177)[name = tensor("use_roll_17")]; - tensor var_1180 = sub(x = var_17, y = use_roll_17)[name = tensor("op_1180")]; - tensor var_1181 = mul(x = filled_window_17, y = var_1180)[name = tensor("op_1181")]; - tensor var_1182 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_1182")]; - tensor input_161 = add(x = var_1181, y = var_1182)[name = tensor("input_161")]; - tensor var_1184 = const()[name = tensor("op_1184"), val = tensor(0x1p+0)]; - tensor var_1185 = add(x = filled_17, y = var_1184)[name = tensor("op_1185")]; - tensor clip_17 = clip(alpha = var_21, beta = var_16, x = var_1185)[name = tensor("clip_17")]; - tensor x_29_axes_0 = const()[name = tensor("x_29_axes_0"), val = tensor([-1])]; - tensor x_29 = layer_norm(axes = x_29_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_161)[name = tensor("x_29")]; - tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_59_pad_type_0 = const()[name = tensor("inputs_59_pad_type_0"), val = tensor("valid")]; - tensor inputs_59_strides_0 = const()[name = tensor("inputs_59_strides_0"), val = tensor([1])]; - tensor inputs_59_pad_0 = const()[name = tensor("inputs_59_pad_0"), val = tensor([0, 0])]; - tensor inputs_59_dilations_0 = const()[name = tensor("inputs_59_dilations_0"), val = tensor([1])]; - tensor inputs_59_groups_0 = const()[name = tensor("inputs_59_groups_0"), val = tensor(1)]; - tensor input_163 = transpose(perm = input_163_perm_0, x = x_29)[name = tensor("transpose_50")]; - tensor inputs_59 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_59_dilations_0, groups = inputs_59_groups_0, pad = inputs_59_pad_0, pad_type = inputs_59_pad_type_0, strides = inputs_59_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_163)[name = tensor("inputs_59")]; - tensor var_1206_split_sizes_0 = const()[name = tensor("op_1206_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1206_axis_0 = const()[name = tensor("op_1206_axis_0"), val = tensor(1)]; - tensor var_1206_0, tensor var_1206_1 = split(axis = var_1206_axis_0, split_sizes = var_1206_split_sizes_0, x = inputs_59)[name = tensor("op_1206")]; - tensor var_1208 = sigmoid(x = var_1206_1)[name = tensor("op_1208")]; - tensor inputs_61 = mul(x = var_1206_0, y = var_1208)[name = tensor("inputs_61")]; - tensor outputs_aug_17_pad_type_0 = const()[name = tensor("outputs_aug_17_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_17_pad_0 = const()[name = tensor("outputs_aug_17_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_17_groups_0 = const()[name = tensor("outputs_aug_17_groups_0"), val = tensor(256)]; - tensor outputs_aug_17_strides_0 = const()[name = tensor("outputs_aug_17_strides_0"), val = tensor([1])]; - tensor outputs_aug_17_dilations_0 = const()[name = tensor("outputs_aug_17_dilations_0"), val = tensor([1])]; - tensor outputs_aug_17 = conv(dilations = outputs_aug_17_dilations_0, groups = outputs_aug_17_groups_0, pad = outputs_aug_17_pad_0, pad_type = outputs_aug_17_pad_type_0, strides = outputs_aug_17_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_61)[name = tensor("outputs_aug_17")]; - tensor input_165_begin_0 = const()[name = tensor("input_165_begin_0"), val = tensor([0, 0, 0])]; - tensor input_165_end_0 = const()[name = tensor("input_165_end_0"), val = tensor([1, 256, 16])]; - tensor input_165_end_mask_0 = const()[name = tensor("input_165_end_mask_0"), val = tensor([true, true, false])]; - tensor input_165 = slice_by_index(begin = input_165_begin_0, end = input_165_end_0, end_mask = input_165_end_mask_0, x = outputs_aug_17)[name = tensor("input_165")]; - tensor inputs_63 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_165)[name = tensor("inputs_63")]; - tensor input_167 = silu(x = inputs_63)[name = tensor("input_167")]; - tensor input_169_pad_type_0 = const()[name = tensor("input_169_pad_type_0"), val = tensor("valid")]; - tensor input_169_strides_0 = const()[name = tensor("input_169_strides_0"), val = tensor([1])]; - tensor input_169_pad_0 = const()[name = tensor("input_169_pad_0"), val = tensor([0, 0])]; - tensor input_169_dilations_0 = const()[name = tensor("input_169_dilations_0"), val = tensor([1])]; - tensor input_169_groups_0 = const()[name = tensor("input_169_groups_0"), val = tensor(1)]; - tensor input_169 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_169_dilations_0, groups = input_169_groups_0, pad = input_169_pad_0, pad_type = input_169_pad_type_0, strides = input_169_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_167)[name = tensor("input_169")]; - tensor conv_out_17_perm_0 = const()[name = tensor("conv_out_17_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1239_begin_0 = const()[name = tensor("op_1239_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1239_end_0 = const()[name = tensor("op_1239_end_0"), val = tensor([1, 16, 256])]; - tensor var_1239_end_mask_0 = const()[name = tensor("op_1239_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_17 = transpose(perm = conv_out_17_perm_0, x = input_169)[name = tensor("transpose_49")]; - tensor var_1239 = slice_by_index(begin = var_1239_begin_0, end = var_1239_end_0, end_mask = var_1239_end_mask_0, x = conv_out_17)[name = tensor("op_1239")]; - tensor var_1242_begin_0 = const()[name = tensor("op_1242_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1242_end_0 = const()[name = tensor("op_1242_end_0"), val = tensor([1, 2, 256])]; - tensor var_1242_end_mask_0 = const()[name = tensor("op_1242_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1242 = slice_by_index(begin = var_1242_begin_0, end = var_1242_end_0, end_mask = var_1242_end_mask_0, x = x_27)[name = tensor("op_1242")]; - tensor var_1245_begin_0 = const()[name = tensor("op_1245_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1245_end_0 = const()[name = tensor("op_1245_end_0"), val = tensor([1, 16, 256])]; - tensor var_1245_end_mask_0 = const()[name = tensor("op_1245_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1245 = slice_by_index(begin = var_1245_begin_0, end = var_1245_end_0, end_mask = var_1245_end_mask_0, x = input_161)[name = tensor("op_1245")]; + tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_836)[name = tensor("x_expanded_17")]; + tensor var_850 = sub(x = var_17, y = fill_mask_17)[name = tensor("op_850")]; + tensor var_851 = mul(x = window_21, y = var_850)[name = tensor("op_851")]; + tensor var_852 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_852")]; + tensor filled_window_17 = add(x = var_851, y = var_852)[name = tensor("filled_window_17")]; + tensor var_854 = greater_equal(x = filled_17, y = var_16)[name = tensor("op_854")]; + tensor var_855_dtype_0 = const()[name = tensor("op_855_dtype_0"), val = tensor("fp32")]; + tensor var_856 = const()[name = tensor("op_856"), val = tensor([1, 1, 1])]; + tensor var_855 = cast(dtype = var_855_dtype_0, x = var_854)[name = tensor("cast_158")]; + tensor use_roll_17 = reshape(shape = var_856, x = var_855)[name = tensor("use_roll_17")]; + tensor var_858 = sub(x = var_17, y = use_roll_17)[name = tensor("op_858")]; + tensor var_859 = mul(x = filled_window_17, y = var_858)[name = tensor("op_859")]; + tensor var_860 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_860")]; + tensor window_23 = add(x = var_859, y = var_860)[name = tensor("window_23")]; + tensor var_862 = const()[name = tensor("op_862"), val = tensor(0x1p+0)]; + tensor var_863 = add(x = filled_17, y = var_862)[name = tensor("op_863")]; + tensor clip_17 = clip(alpha = var_21, beta = var_16, x = var_863)[name = tensor("clip_17")]; + tensor var_866_begin_0 = const()[name = tensor("op_866_begin_0"), val = tensor([0, 1, 0])]; + tensor var_866_end_0 = const()[name = tensor("op_866_end_0"), val = tensor([1, 2, 256])]; + tensor var_866_end_mask_0 = const()[name = tensor("op_866_end_mask_0"), val = tensor([true, false, true])]; + tensor var_866 = slice_by_index(begin = var_866_begin_0, end = var_866_end_0, end_mask = var_866_end_mask_0, x = x_15)[name = tensor("op_866")]; + tensor var_869_begin_0 = const()[name = tensor("op_869_begin_0"), val = tensor([0, 1, 0])]; + tensor var_869_end_0 = const()[name = tensor("op_869_end_0"), val = tensor([1, 16, 256])]; + tensor var_869_end_mask_0 = const()[name = tensor("op_869_end_mask_0"), val = tensor([true, true, true])]; + tensor var_869 = slice_by_index(begin = var_869_begin_0, end = var_869_end_0, end_mask = var_869_end_mask_0, x = window_23)[name = tensor("op_869")]; tensor rolled_19_interleave_0 = const()[name = tensor("rolled_19_interleave_0"), val = tensor(false)]; - tensor rolled_19 = concat(axis = var_33, interleave = rolled_19_interleave_0, values = (var_1245, var_1242))[name = tensor("rolled_19")]; + tensor rolled_19 = concat(axis = var_33, interleave = rolled_19_interleave_0, values = (var_869, var_866))[name = tensor("rolled_19")]; tensor clip_18 = clip(alpha = var_21, beta = var_20, x = clip_17)[name = tensor("clip_18")]; - tensor var_1250 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_1250")]; - tensor var_1251_dtype_0 = const()[name = tensor("op_1251_dtype_0"), val = tensor("fp32")]; - tensor var_1252 = const()[name = tensor("op_1252"), val = tensor([1, 16, 1])]; - tensor var_1251 = cast(dtype = var_1251_dtype_0, x = var_1250)[name = tensor("cast_157")]; - tensor fill_mask_19 = reshape(shape = var_1252, x = var_1251)[name = tensor("fill_mask_19")]; + tensor var_874 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_874")]; + tensor var_875_dtype_0 = const()[name = tensor("op_875_dtype_0"), val = tensor("fp32")]; + tensor var_876 = const()[name = tensor("op_876"), val = tensor([1, 16, 1])]; + tensor var_875 = cast(dtype = var_875_dtype_0, x = var_874)[name = tensor("cast_157")]; + tensor fill_mask_19 = reshape(shape = var_876, x = var_875)[name = tensor("fill_mask_19")]; tensor x_expanded_19_reps_0 = const()[name = tensor("x_expanded_19_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_1242)[name = tensor("x_expanded_19")]; - tensor var_1256 = sub(x = var_17, y = fill_mask_19)[name = tensor("op_1256")]; - tensor var_1257 = mul(x = input_161, y = var_1256)[name = tensor("op_1257")]; - tensor var_1258 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_1258")]; - tensor filled_window_19 = add(x = var_1257, y = var_1258)[name = tensor("filled_window_19")]; - tensor var_1260 = greater_equal(x = clip_17, y = var_16)[name = tensor("op_1260")]; - tensor var_1261_dtype_0 = const()[name = tensor("op_1261_dtype_0"), val = tensor("fp32")]; - tensor var_1262 = const()[name = tensor("op_1262"), val = tensor([1, 1, 1])]; - tensor var_1261 = cast(dtype = var_1261_dtype_0, x = var_1260)[name = tensor("cast_156")]; - tensor use_roll_19 = reshape(shape = var_1262, x = var_1261)[name = tensor("use_roll_19")]; - tensor var_1264 = sub(x = var_17, y = use_roll_19)[name = tensor("op_1264")]; - tensor var_1265 = mul(x = filled_window_19, y = var_1264)[name = tensor("op_1265")]; - tensor var_1266 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_1266")]; - tensor input_171 = add(x = var_1265, y = var_1266)[name = tensor("input_171")]; - tensor var_1268 = const()[name = tensor("op_1268"), val = tensor(0x1p+0)]; - tensor var_1269 = add(x = clip_17, y = var_1268)[name = tensor("op_1269")]; - tensor clip_19 = clip(alpha = var_21, beta = var_16, x = var_1269)[name = tensor("clip_19")]; - tensor x_31_axes_0 = const()[name = tensor("x_31_axes_0"), val = tensor([-1])]; - tensor x_31 = layer_norm(axes = x_31_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_171)[name = tensor("x_31")]; - tensor input_173_perm_0 = const()[name = tensor("input_173_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_65_pad_type_0 = const()[name = tensor("inputs_65_pad_type_0"), val = tensor("valid")]; - tensor inputs_65_strides_0 = const()[name = tensor("inputs_65_strides_0"), val = tensor([1])]; - tensor inputs_65_pad_0 = const()[name = tensor("inputs_65_pad_0"), val = tensor([0, 0])]; - tensor inputs_65_dilations_0 = const()[name = tensor("inputs_65_dilations_0"), val = tensor([1])]; - tensor inputs_65_groups_0 = const()[name = tensor("inputs_65_groups_0"), val = tensor(1)]; - tensor input_173 = transpose(perm = input_173_perm_0, x = x_31)[name = tensor("transpose_48")]; - tensor inputs_65 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_65_dilations_0, groups = inputs_65_groups_0, pad = inputs_65_pad_0, pad_type = inputs_65_pad_type_0, strides = inputs_65_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_173)[name = tensor("inputs_65")]; - tensor var_1290_split_sizes_0 = const()[name = tensor("op_1290_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1290_axis_0 = const()[name = tensor("op_1290_axis_0"), val = tensor(1)]; - tensor var_1290_0, tensor var_1290_1 = split(axis = var_1290_axis_0, split_sizes = var_1290_split_sizes_0, x = inputs_65)[name = tensor("op_1290")]; - tensor var_1292 = sigmoid(x = var_1290_1)[name = tensor("op_1292")]; - tensor inputs_67 = mul(x = var_1290_0, y = var_1292)[name = tensor("inputs_67")]; - tensor outputs_aug_19_pad_type_0 = const()[name = tensor("outputs_aug_19_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_19_pad_0 = const()[name = tensor("outputs_aug_19_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_19_groups_0 = const()[name = tensor("outputs_aug_19_groups_0"), val = tensor(256)]; - tensor outputs_aug_19_strides_0 = const()[name = tensor("outputs_aug_19_strides_0"), val = tensor([1])]; - tensor outputs_aug_19_dilations_0 = const()[name = tensor("outputs_aug_19_dilations_0"), val = tensor([1])]; - tensor outputs_aug_19 = conv(dilations = outputs_aug_19_dilations_0, groups = outputs_aug_19_groups_0, pad = outputs_aug_19_pad_0, pad_type = outputs_aug_19_pad_type_0, strides = outputs_aug_19_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_67)[name = tensor("outputs_aug_19")]; - tensor input_175_begin_0 = const()[name = tensor("input_175_begin_0"), val = tensor([0, 0, 0])]; - tensor input_175_end_0 = const()[name = tensor("input_175_end_0"), val = tensor([1, 256, 16])]; - tensor input_175_end_mask_0 = const()[name = tensor("input_175_end_mask_0"), val = tensor([true, true, false])]; - tensor input_175 = slice_by_index(begin = input_175_begin_0, end = input_175_end_0, end_mask = input_175_end_mask_0, x = outputs_aug_19)[name = tensor("input_175")]; - tensor inputs_69 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_175)[name = tensor("inputs_69")]; - tensor input_177 = silu(x = inputs_69)[name = tensor("input_177")]; - tensor input_179_pad_type_0 = const()[name = tensor("input_179_pad_type_0"), val = tensor("valid")]; - tensor input_179_strides_0 = const()[name = tensor("input_179_strides_0"), val = tensor([1])]; - tensor input_179_pad_0 = const()[name = tensor("input_179_pad_0"), val = tensor([0, 0])]; - tensor input_179_dilations_0 = const()[name = tensor("input_179_dilations_0"), val = tensor([1])]; - tensor input_179_groups_0 = const()[name = tensor("input_179_groups_0"), val = tensor(1)]; - tensor input_179 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_179_dilations_0, groups = input_179_groups_0, pad = input_179_pad_0, pad_type = input_179_pad_type_0, strides = input_179_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_177)[name = tensor("input_179")]; - tensor conv_out_19_perm_0 = const()[name = tensor("conv_out_19_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1323_begin_0 = const()[name = tensor("op_1323_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1323_end_0 = const()[name = tensor("op_1323_end_0"), val = tensor([1, 16, 256])]; - tensor var_1323_end_mask_0 = const()[name = tensor("op_1323_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_19 = transpose(perm = conv_out_19_perm_0, x = input_179)[name = tensor("transpose_47")]; - tensor var_1323 = slice_by_index(begin = var_1323_begin_0, end = var_1323_end_0, end_mask = var_1323_end_mask_0, x = conv_out_19)[name = tensor("op_1323")]; - tensor var_1326_begin_0 = const()[name = tensor("op_1326_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1326_end_0 = const()[name = tensor("op_1326_end_0"), val = tensor([1, 3, 256])]; - tensor var_1326_end_mask_0 = const()[name = tensor("op_1326_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1326 = slice_by_index(begin = var_1326_begin_0, end = var_1326_end_0, end_mask = var_1326_end_mask_0, x = x_27)[name = tensor("op_1326")]; - tensor var_1329_begin_0 = const()[name = tensor("op_1329_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1329_end_0 = const()[name = tensor("op_1329_end_0"), val = tensor([1, 16, 256])]; - tensor var_1329_end_mask_0 = const()[name = tensor("op_1329_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1329 = slice_by_index(begin = var_1329_begin_0, end = var_1329_end_0, end_mask = var_1329_end_mask_0, x = input_171)[name = tensor("op_1329")]; + tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_866)[name = tensor("x_expanded_19")]; + tensor var_880 = sub(x = var_17, y = fill_mask_19)[name = tensor("op_880")]; + tensor var_881 = mul(x = window_23, y = var_880)[name = tensor("op_881")]; + tensor var_882 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_882")]; + tensor filled_window_19 = add(x = var_881, y = var_882)[name = tensor("filled_window_19")]; + tensor var_884 = greater_equal(x = clip_17, y = var_16)[name = tensor("op_884")]; + tensor var_885_dtype_0 = const()[name = tensor("op_885_dtype_0"), val = tensor("fp32")]; + tensor var_886 = const()[name = tensor("op_886"), val = tensor([1, 1, 1])]; + tensor var_885 = cast(dtype = var_885_dtype_0, x = var_884)[name = tensor("cast_156")]; + tensor use_roll_19 = reshape(shape = var_886, x = var_885)[name = tensor("use_roll_19")]; + tensor var_888 = sub(x = var_17, y = use_roll_19)[name = tensor("op_888")]; + tensor var_889 = mul(x = filled_window_19, y = var_888)[name = tensor("op_889")]; + tensor var_890 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_890")]; + tensor window_25 = add(x = var_889, y = var_890)[name = tensor("window_25")]; + tensor var_892 = const()[name = tensor("op_892"), val = tensor(0x1p+0)]; + tensor var_893 = add(x = clip_17, y = var_892)[name = tensor("op_893")]; + tensor clip_19 = clip(alpha = var_21, beta = var_16, x = var_893)[name = tensor("clip_19")]; + tensor var_896_begin_0 = const()[name = tensor("op_896_begin_0"), val = tensor([0, 2, 0])]; + tensor var_896_end_0 = const()[name = tensor("op_896_end_0"), val = tensor([1, 3, 256])]; + tensor var_896_end_mask_0 = const()[name = tensor("op_896_end_mask_0"), val = tensor([true, false, true])]; + tensor var_896 = slice_by_index(begin = var_896_begin_0, end = var_896_end_0, end_mask = var_896_end_mask_0, x = x_15)[name = tensor("op_896")]; + tensor var_899_begin_0 = const()[name = tensor("op_899_begin_0"), val = tensor([0, 1, 0])]; + tensor var_899_end_0 = const()[name = tensor("op_899_end_0"), val = tensor([1, 16, 256])]; + tensor var_899_end_mask_0 = const()[name = tensor("op_899_end_mask_0"), val = tensor([true, true, true])]; + tensor var_899 = slice_by_index(begin = var_899_begin_0, end = var_899_end_0, end_mask = var_899_end_mask_0, x = window_25)[name = tensor("op_899")]; tensor rolled_21_interleave_0 = const()[name = tensor("rolled_21_interleave_0"), val = tensor(false)]; - tensor rolled_21 = concat(axis = var_33, interleave = rolled_21_interleave_0, values = (var_1329, var_1326))[name = tensor("rolled_21")]; + tensor rolled_21 = concat(axis = var_33, interleave = rolled_21_interleave_0, values = (var_899, var_896))[name = tensor("rolled_21")]; tensor clip_20 = clip(alpha = var_21, beta = var_20, x = clip_19)[name = tensor("clip_20")]; - tensor var_1334 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_1334")]; - tensor var_1335_dtype_0 = const()[name = tensor("op_1335_dtype_0"), val = tensor("fp32")]; - tensor var_1336 = const()[name = tensor("op_1336"), val = tensor([1, 16, 1])]; - tensor var_1335 = cast(dtype = var_1335_dtype_0, x = var_1334)[name = tensor("cast_155")]; - tensor fill_mask_21 = reshape(shape = var_1336, x = var_1335)[name = tensor("fill_mask_21")]; + tensor var_904 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_904")]; + tensor var_905_dtype_0 = const()[name = tensor("op_905_dtype_0"), val = tensor("fp32")]; + tensor var_906 = const()[name = tensor("op_906"), val = tensor([1, 16, 1])]; + tensor var_905 = cast(dtype = var_905_dtype_0, x = var_904)[name = tensor("cast_155")]; + tensor fill_mask_21 = reshape(shape = var_906, x = var_905)[name = tensor("fill_mask_21")]; tensor x_expanded_21_reps_0 = const()[name = tensor("x_expanded_21_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_1326)[name = tensor("x_expanded_21")]; - tensor var_1340 = sub(x = var_17, y = fill_mask_21)[name = tensor("op_1340")]; - tensor var_1341 = mul(x = input_171, y = var_1340)[name = tensor("op_1341")]; - tensor var_1342 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_1342")]; - tensor filled_window_21 = add(x = var_1341, y = var_1342)[name = tensor("filled_window_21")]; - tensor var_1344 = greater_equal(x = clip_19, y = var_16)[name = tensor("op_1344")]; - tensor var_1345_dtype_0 = const()[name = tensor("op_1345_dtype_0"), val = tensor("fp32")]; - tensor var_1346 = const()[name = tensor("op_1346"), val = tensor([1, 1, 1])]; - tensor var_1345 = cast(dtype = var_1345_dtype_0, x = var_1344)[name = tensor("cast_154")]; - tensor use_roll_21 = reshape(shape = var_1346, x = var_1345)[name = tensor("use_roll_21")]; - tensor var_1348 = sub(x = var_17, y = use_roll_21)[name = tensor("op_1348")]; - tensor var_1349 = mul(x = filled_window_21, y = var_1348)[name = tensor("op_1349")]; - tensor var_1350 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_1350")]; - tensor input_181 = add(x = var_1349, y = var_1350)[name = tensor("input_181")]; - tensor var_1352 = const()[name = tensor("op_1352"), val = tensor(0x1p+0)]; - tensor var_1353 = add(x = clip_19, y = var_1352)[name = tensor("op_1353")]; - tensor clip_21 = clip(alpha = var_21, beta = var_16, x = var_1353)[name = tensor("clip_21")]; - tensor x_33_axes_0 = const()[name = tensor("x_33_axes_0"), val = tensor([-1])]; - tensor x_33 = layer_norm(axes = x_33_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_181)[name = tensor("x_33")]; - tensor input_183_perm_0 = const()[name = tensor("input_183_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_71_pad_type_0 = const()[name = tensor("inputs_71_pad_type_0"), val = tensor("valid")]; - tensor inputs_71_strides_0 = const()[name = tensor("inputs_71_strides_0"), val = tensor([1])]; - tensor inputs_71_pad_0 = const()[name = tensor("inputs_71_pad_0"), val = tensor([0, 0])]; - tensor inputs_71_dilations_0 = const()[name = tensor("inputs_71_dilations_0"), val = tensor([1])]; - tensor inputs_71_groups_0 = const()[name = tensor("inputs_71_groups_0"), val = tensor(1)]; - tensor input_183 = transpose(perm = input_183_perm_0, x = x_33)[name = tensor("transpose_46")]; - tensor inputs_71 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_71_dilations_0, groups = inputs_71_groups_0, pad = inputs_71_pad_0, pad_type = inputs_71_pad_type_0, strides = inputs_71_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_183)[name = tensor("inputs_71")]; - tensor var_1374_split_sizes_0 = const()[name = tensor("op_1374_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1374_axis_0 = const()[name = tensor("op_1374_axis_0"), val = tensor(1)]; - tensor var_1374_0, tensor var_1374_1 = split(axis = var_1374_axis_0, split_sizes = var_1374_split_sizes_0, x = inputs_71)[name = tensor("op_1374")]; - tensor var_1376 = sigmoid(x = var_1374_1)[name = tensor("op_1376")]; - tensor inputs_73 = mul(x = var_1374_0, y = var_1376)[name = tensor("inputs_73")]; - tensor outputs_aug_21_pad_type_0 = const()[name = tensor("outputs_aug_21_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_21_pad_0 = const()[name = tensor("outputs_aug_21_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_21_groups_0 = const()[name = tensor("outputs_aug_21_groups_0"), val = tensor(256)]; - tensor outputs_aug_21_strides_0 = const()[name = tensor("outputs_aug_21_strides_0"), val = tensor([1])]; - tensor outputs_aug_21_dilations_0 = const()[name = tensor("outputs_aug_21_dilations_0"), val = tensor([1])]; - tensor outputs_aug_21 = conv(dilations = outputs_aug_21_dilations_0, groups = outputs_aug_21_groups_0, pad = outputs_aug_21_pad_0, pad_type = outputs_aug_21_pad_type_0, strides = outputs_aug_21_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_73)[name = tensor("outputs_aug_21")]; - tensor input_185_begin_0 = const()[name = tensor("input_185_begin_0"), val = tensor([0, 0, 0])]; - tensor input_185_end_0 = const()[name = tensor("input_185_end_0"), val = tensor([1, 256, 16])]; - tensor input_185_end_mask_0 = const()[name = tensor("input_185_end_mask_0"), val = tensor([true, true, false])]; - tensor input_185 = slice_by_index(begin = input_185_begin_0, end = input_185_end_0, end_mask = input_185_end_mask_0, x = outputs_aug_21)[name = tensor("input_185")]; - tensor inputs_75 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_185)[name = tensor("inputs_75")]; - tensor input_187 = silu(x = inputs_75)[name = tensor("input_187")]; - tensor input_189_pad_type_0 = const()[name = tensor("input_189_pad_type_0"), val = tensor("valid")]; - tensor input_189_strides_0 = const()[name = tensor("input_189_strides_0"), val = tensor([1])]; - tensor input_189_pad_0 = const()[name = tensor("input_189_pad_0"), val = tensor([0, 0])]; - tensor input_189_dilations_0 = const()[name = tensor("input_189_dilations_0"), val = tensor([1])]; - tensor input_189_groups_0 = const()[name = tensor("input_189_groups_0"), val = tensor(1)]; - tensor input_189 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_189_dilations_0, groups = input_189_groups_0, pad = input_189_pad_0, pad_type = input_189_pad_type_0, strides = input_189_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_187)[name = tensor("input_189")]; - tensor conv_out_21_perm_0 = const()[name = tensor("conv_out_21_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1407_begin_0 = const()[name = tensor("op_1407_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1407_end_0 = const()[name = tensor("op_1407_end_0"), val = tensor([1, 16, 256])]; - tensor var_1407_end_mask_0 = const()[name = tensor("op_1407_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_21 = transpose(perm = conv_out_21_perm_0, x = input_189)[name = tensor("transpose_45")]; - tensor var_1407 = slice_by_index(begin = var_1407_begin_0, end = var_1407_end_0, end_mask = var_1407_end_mask_0, x = conv_out_21)[name = tensor("op_1407")]; - tensor var_1410_begin_0 = const()[name = tensor("op_1410_begin_0"), val = tensor([0, 3, 0])]; - tensor var_1410_end_0 = const()[name = tensor("op_1410_end_0"), val = tensor([1, 1, 256])]; - tensor var_1410_end_mask_0 = const()[name = tensor("op_1410_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1410 = slice_by_index(begin = var_1410_begin_0, end = var_1410_end_0, end_mask = var_1410_end_mask_0, x = x_27)[name = tensor("op_1410")]; - tensor var_1413_begin_0 = const()[name = tensor("op_1413_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1413_end_0 = const()[name = tensor("op_1413_end_0"), val = tensor([1, 16, 256])]; - tensor var_1413_end_mask_0 = const()[name = tensor("op_1413_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1413 = slice_by_index(begin = var_1413_begin_0, end = var_1413_end_0, end_mask = var_1413_end_mask_0, x = input_181)[name = tensor("op_1413")]; + tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_896)[name = tensor("x_expanded_21")]; + tensor var_910 = sub(x = var_17, y = fill_mask_21)[name = tensor("op_910")]; + tensor var_911 = mul(x = window_25, y = var_910)[name = tensor("op_911")]; + tensor var_912 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_912")]; + tensor filled_window_21 = add(x = var_911, y = var_912)[name = tensor("filled_window_21")]; + tensor var_914 = greater_equal(x = clip_19, y = var_16)[name = tensor("op_914")]; + tensor var_915_dtype_0 = const()[name = tensor("op_915_dtype_0"), val = tensor("fp32")]; + tensor var_916 = const()[name = tensor("op_916"), val = tensor([1, 1, 1])]; + tensor var_915 = cast(dtype = var_915_dtype_0, x = var_914)[name = tensor("cast_154")]; + tensor use_roll_21 = reshape(shape = var_916, x = var_915)[name = tensor("use_roll_21")]; + tensor var_918 = sub(x = var_17, y = use_roll_21)[name = tensor("op_918")]; + tensor var_919 = mul(x = filled_window_21, y = var_918)[name = tensor("op_919")]; + tensor var_920 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_920")]; + tensor window_27 = add(x = var_919, y = var_920)[name = tensor("window_27")]; + tensor var_922 = const()[name = tensor("op_922"), val = tensor(0x1p+0)]; + tensor var_923 = add(x = clip_19, y = var_922)[name = tensor("op_923")]; + tensor clip_21 = clip(alpha = var_21, beta = var_16, x = var_923)[name = tensor("clip_21")]; + tensor var_926_begin_0 = const()[name = tensor("op_926_begin_0"), val = tensor([0, 3, 0])]; + tensor var_926_end_0 = const()[name = tensor("op_926_end_0"), val = tensor([1, 1, 256])]; + tensor var_926_end_mask_0 = const()[name = tensor("op_926_end_mask_0"), val = tensor([true, true, true])]; + tensor var_926 = slice_by_index(begin = var_926_begin_0, end = var_926_end_0, end_mask = var_926_end_mask_0, x = x_15)[name = tensor("op_926")]; + tensor var_929_begin_0 = const()[name = tensor("op_929_begin_0"), val = tensor([0, 1, 0])]; + tensor var_929_end_0 = const()[name = tensor("op_929_end_0"), val = tensor([1, 16, 256])]; + tensor var_929_end_mask_0 = const()[name = tensor("op_929_end_mask_0"), val = tensor([true, true, true])]; + tensor var_929 = slice_by_index(begin = var_929_begin_0, end = var_929_end_0, end_mask = var_929_end_mask_0, x = window_27)[name = tensor("op_929")]; tensor rolled_23_interleave_0 = const()[name = tensor("rolled_23_interleave_0"), val = tensor(false)]; - tensor rolled_23 = concat(axis = var_33, interleave = rolled_23_interleave_0, values = (var_1413, var_1410))[name = tensor("rolled_23")]; + tensor rolled_23 = concat(axis = var_33, interleave = rolled_23_interleave_0, values = (var_929, var_926))[name = tensor("rolled_23")]; tensor clip_22 = clip(alpha = var_21, beta = var_20, x = clip_21)[name = tensor("clip_22")]; - tensor var_1418 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_1418")]; - tensor var_1419_dtype_0 = const()[name = tensor("op_1419_dtype_0"), val = tensor("fp32")]; - tensor var_1420 = const()[name = tensor("op_1420"), val = tensor([1, 16, 1])]; - tensor var_1419 = cast(dtype = var_1419_dtype_0, x = var_1418)[name = tensor("cast_153")]; - tensor fill_mask_23 = reshape(shape = var_1420, x = var_1419)[name = tensor("fill_mask_23")]; + tensor var_934 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_934")]; + tensor var_935_dtype_0 = const()[name = tensor("op_935_dtype_0"), val = tensor("fp32")]; + tensor var_936 = const()[name = tensor("op_936"), val = tensor([1, 16, 1])]; + tensor var_935 = cast(dtype = var_935_dtype_0, x = var_934)[name = tensor("cast_153")]; + tensor fill_mask_23 = reshape(shape = var_936, x = var_935)[name = tensor("fill_mask_23")]; tensor x_expanded_23_reps_0 = const()[name = tensor("x_expanded_23_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_23 = tile(reps = x_expanded_23_reps_0, x = var_1410)[name = tensor("x_expanded_23")]; - tensor var_1424 = sub(x = var_17, y = fill_mask_23)[name = tensor("op_1424")]; - tensor var_1425 = mul(x = input_181, y = var_1424)[name = tensor("op_1425")]; - tensor var_1426 = mul(x = x_expanded_23, y = fill_mask_23)[name = tensor("op_1426")]; - tensor filled_window_23 = add(x = var_1425, y = var_1426)[name = tensor("filled_window_23")]; - tensor var_1428 = greater_equal(x = clip_21, y = var_16)[name = tensor("op_1428")]; - tensor var_1429_dtype_0 = const()[name = tensor("op_1429_dtype_0"), val = tensor("fp32")]; - tensor var_1430 = const()[name = tensor("op_1430"), val = tensor([1, 1, 1])]; - tensor var_1429 = cast(dtype = var_1429_dtype_0, x = var_1428)[name = tensor("cast_152")]; - tensor use_roll_23 = reshape(shape = var_1430, x = var_1429)[name = tensor("use_roll_23")]; - tensor var_1432 = sub(x = var_17, y = use_roll_23)[name = tensor("op_1432")]; - tensor var_1433 = mul(x = filled_window_23, y = var_1432)[name = tensor("op_1433")]; - tensor var_1434 = mul(x = rolled_23, y = use_roll_23)[name = tensor("op_1434")]; - tensor input_191 = add(x = var_1433, y = var_1434)[name = tensor("input_191")]; - tensor var_1436 = const()[name = tensor("op_1436"), val = tensor(0x1p+0)]; - tensor var_1437 = add(x = clip_21, y = var_1436)[name = tensor("op_1437")]; - tensor clip_23 = clip(alpha = var_21, beta = var_16, x = var_1437)[name = tensor("clip_23")]; - tensor x_35_axes_0 = const()[name = tensor("x_35_axes_0"), val = tensor([-1])]; - tensor x_35 = layer_norm(axes = x_35_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_191)[name = tensor("x_35")]; - tensor input_193_perm_0 = const()[name = tensor("input_193_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_77_pad_type_0 = const()[name = tensor("inputs_77_pad_type_0"), val = tensor("valid")]; - tensor inputs_77_strides_0 = const()[name = tensor("inputs_77_strides_0"), val = tensor([1])]; - tensor inputs_77_pad_0 = const()[name = tensor("inputs_77_pad_0"), val = tensor([0, 0])]; - tensor inputs_77_dilations_0 = const()[name = tensor("inputs_77_dilations_0"), val = tensor([1])]; - tensor inputs_77_groups_0 = const()[name = tensor("inputs_77_groups_0"), val = tensor(1)]; - tensor input_193 = transpose(perm = input_193_perm_0, x = x_35)[name = tensor("transpose_44")]; - tensor inputs_77 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_77_dilations_0, groups = inputs_77_groups_0, pad = inputs_77_pad_0, pad_type = inputs_77_pad_type_0, strides = inputs_77_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_193)[name = tensor("inputs_77")]; - tensor var_1458_split_sizes_0 = const()[name = tensor("op_1458_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1458_axis_0 = const()[name = tensor("op_1458_axis_0"), val = tensor(1)]; - tensor var_1458_0, tensor var_1458_1 = split(axis = var_1458_axis_0, split_sizes = var_1458_split_sizes_0, x = inputs_77)[name = tensor("op_1458")]; - tensor var_1460 = sigmoid(x = var_1458_1)[name = tensor("op_1460")]; - tensor inputs_79 = mul(x = var_1458_0, y = var_1460)[name = tensor("inputs_79")]; - tensor outputs_aug_23_pad_type_0 = const()[name = tensor("outputs_aug_23_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_23_pad_0 = const()[name = tensor("outputs_aug_23_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_23_groups_0 = const()[name = tensor("outputs_aug_23_groups_0"), val = tensor(256)]; - tensor outputs_aug_23_strides_0 = const()[name = tensor("outputs_aug_23_strides_0"), val = tensor([1])]; - tensor outputs_aug_23_dilations_0 = const()[name = tensor("outputs_aug_23_dilations_0"), val = tensor([1])]; - tensor outputs_aug_23 = conv(dilations = outputs_aug_23_dilations_0, groups = outputs_aug_23_groups_0, pad = outputs_aug_23_pad_0, pad_type = outputs_aug_23_pad_type_0, strides = outputs_aug_23_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_79)[name = tensor("outputs_aug_23")]; - tensor input_195_begin_0 = const()[name = tensor("input_195_begin_0"), val = tensor([0, 0, 0])]; - tensor input_195_end_0 = const()[name = tensor("input_195_end_0"), val = tensor([1, 256, 16])]; - tensor input_195_end_mask_0 = const()[name = tensor("input_195_end_mask_0"), val = tensor([true, true, false])]; - tensor input_195 = slice_by_index(begin = input_195_begin_0, end = input_195_end_0, end_mask = input_195_end_mask_0, x = outputs_aug_23)[name = tensor("input_195")]; - tensor inputs_81 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_195)[name = tensor("inputs_81")]; - tensor input_197 = silu(x = inputs_81)[name = tensor("input_197")]; - tensor input_199_pad_type_0 = const()[name = tensor("input_199_pad_type_0"), val = tensor("valid")]; - tensor input_199_strides_0 = const()[name = tensor("input_199_strides_0"), val = tensor([1])]; - tensor input_199_pad_0 = const()[name = tensor("input_199_pad_0"), val = tensor([0, 0])]; - tensor input_199_dilations_0 = const()[name = tensor("input_199_dilations_0"), val = tensor([1])]; - tensor input_199_groups_0 = const()[name = tensor("input_199_groups_0"), val = tensor(1)]; - tensor input_199 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_199_dilations_0, groups = input_199_groups_0, pad = input_199_pad_0, pad_type = input_199_pad_type_0, strides = input_199_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_197)[name = tensor("input_199")]; - tensor conv_out_23_perm_0 = const()[name = tensor("conv_out_23_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1491_begin_0 = const()[name = tensor("op_1491_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1491_end_0 = const()[name = tensor("op_1491_end_0"), val = tensor([1, 16, 256])]; - tensor var_1491_end_mask_0 = const()[name = tensor("op_1491_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_23 = transpose(perm = conv_out_23_perm_0, x = input_199)[name = tensor("transpose_43")]; - tensor var_1491 = slice_by_index(begin = var_1491_begin_0, end = var_1491_end_0, end_mask = var_1491_end_mask_0, x = conv_out_23)[name = tensor("op_1491")]; - tensor var_1494_interleave_0 = const()[name = tensor("op_1494_interleave_0"), val = tensor(false)]; - tensor var_1494 = concat(axis = var_33, interleave = var_1494_interleave_0, values = (var_1239, var_1323, var_1407, var_1491))[name = tensor("op_1494")]; - tensor input_201 = add(x = x_27, y = var_1494)[name = tensor("input_201")]; - tensor input_203_axes_0 = const()[name = tensor("input_203_axes_0"), val = tensor([-1])]; - tensor input_203 = layer_norm(axes = input_203_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_201)[name = tensor("input_203")]; - tensor inputs_83 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_203)[name = tensor("linear_26")]; - tensor input_205 = silu(x = inputs_83)[name = tensor("input_205")]; - tensor input_209 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_205)[name = tensor("linear_27")]; - tensor var_1517 = const()[name = tensor("op_1517"), val = tensor(0x1p-1)]; - tensor var_1518 = mul(x = input_209, y = var_1517)[name = tensor("op_1518")]; - tensor input_211 = add(x = var_1518, y = input_201)[name = tensor("input_211")]; - tensor input_213_axes_0 = const()[name = tensor("input_213_axes_0"), val = tensor([-1])]; - tensor input_213 = layer_norm(axes = input_213_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_211)[name = tensor("input_213")]; - tensor input_215_axes_0 = const()[name = tensor("input_215_axes_0"), val = tensor([-1])]; - tensor input_215 = layer_norm(axes = input_215_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_213)[name = tensor("input_215")]; - tensor inputs_85 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_215)[name = tensor("linear_28")]; - tensor input_217 = silu(x = inputs_85)[name = tensor("input_217")]; - tensor input_221 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_217)[name = tensor("linear_29")]; - tensor var_1547 = const()[name = tensor("op_1547"), val = tensor(0x1p-1)]; - tensor var_1548 = mul(x = input_221, y = var_1547)[name = tensor("op_1548")]; - tensor input_223 = add(x = var_1548, y = input_213)[name = tensor("input_223")]; - tensor x_37_axes_0 = const()[name = tensor("x_37_axes_0"), val = tensor([-1])]; - tensor x_37 = layer_norm(axes = x_37_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_223)[name = tensor("x_37")]; + tensor x_expanded_23 = tile(reps = x_expanded_23_reps_0, x = var_926)[name = tensor("x_expanded_23")]; + tensor var_940 = sub(x = var_17, y = fill_mask_23)[name = tensor("op_940")]; + tensor var_941 = mul(x = window_27, y = var_940)[name = tensor("op_941")]; + tensor var_942 = mul(x = x_expanded_23, y = fill_mask_23)[name = tensor("op_942")]; + tensor filled_window_23 = add(x = var_941, y = var_942)[name = tensor("filled_window_23")]; + tensor var_944 = greater_equal(x = clip_21, y = var_16)[name = tensor("op_944")]; + tensor var_945_dtype_0 = const()[name = tensor("op_945_dtype_0"), val = tensor("fp32")]; + tensor var_946 = const()[name = tensor("op_946"), val = tensor([1, 1, 1])]; + tensor var_945 = cast(dtype = var_945_dtype_0, x = var_944)[name = tensor("cast_152")]; + tensor use_roll_23 = reshape(shape = var_946, x = var_945)[name = tensor("use_roll_23")]; + tensor var_948 = sub(x = var_17, y = use_roll_23)[name = tensor("op_948")]; + tensor var_949 = mul(x = filled_window_23, y = var_948)[name = tensor("op_949")]; + tensor var_950 = mul(x = rolled_23, y = use_roll_23)[name = tensor("op_950")]; + tensor window_29 = add(x = var_949, y = var_950)[name = tensor("window_29")]; + tensor var_952 = const()[name = tensor("op_952"), val = tensor(0x1p+0)]; + tensor var_953 = add(x = clip_21, y = var_952)[name = tensor("op_953")]; + tensor clip_23 = clip(alpha = var_21, beta = var_16, x = var_953)[name = tensor("clip_23")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_30, interleave = input_101_interleave_0, values = (window_23, window_25, window_27, window_29))[name = tensor("input_101")]; + tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; + tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; + tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_23_pad_type_0 = const()[name = tensor("inputs_23_pad_type_0"), val = tensor("valid")]; + tensor inputs_23_strides_0 = const()[name = tensor("inputs_23_strides_0"), val = tensor([1])]; + tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; + tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; + tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; + tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; + tensor var_976_split_sizes_0 = const()[name = tensor("op_976_split_sizes_0"), val = tensor([256, 256])]; + tensor var_976_axis_0 = const()[name = tensor("op_976_axis_0"), val = tensor(1)]; + tensor var_976_0, tensor var_976_1 = split(axis = var_976_axis_0, split_sizes = var_976_split_sizes_0, x = inputs_23)[name = tensor("op_976")]; + tensor var_978 = sigmoid(x = var_976_1)[name = tensor("op_978")]; + tensor inputs_25 = mul(x = var_976_0, y = var_978)[name = tensor("inputs_25")]; + tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; + tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; + tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; + tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_25)[name = tensor("outputs_aug_5")]; + tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; + tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([4, 256, 16])]; + tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; + tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_5)[name = tensor("input_105")]; + tensor inputs_27 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_105)[name = tensor("inputs_27")]; + tensor input_107 = silu(x = inputs_27)[name = tensor("input_107")]; + tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; + tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; + tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; + tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; + tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; + tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; + tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1009_begin_0 = const()[name = tensor("op_1009_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1009_end_0 = const()[name = tensor("op_1009_end_0"), val = tensor([4, 16, 256])]; + tensor var_1009_end_mask_0 = const()[name = tensor("op_1009_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_1009 = slice_by_index(begin = var_1009_begin_0, end = var_1009_end_0, end_mask = var_1009_end_mask_0, x = conv_out_5)[name = tensor("op_1009")]; + tensor var_1011_perm_0 = const()[name = tensor("op_1011_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1011 = transpose(perm = var_1011_perm_0, x = var_1009)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_1011)[name = tensor("input_111")]; + tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; + tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; + tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; + tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; + tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; + tensor var_1034 = const()[name = tensor("op_1034"), val = tensor(0x1p-1)]; + tensor var_1035 = mul(x = input_119, y = var_1034)[name = tensor("op_1035")]; + tensor input_121 = add(x = var_1035, y = input_111)[name = tensor("input_121")]; + tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; + tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_35, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; + tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; + tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; + tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; + tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; + tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; + tensor var_1064 = const()[name = tensor("op_1064"), val = tensor(0x1p-1)]; + tensor var_1065 = mul(x = input_131, y = var_1064)[name = tensor("op_1065")]; + tensor input_133 = add(x = var_1065, y = input_123)[name = tensor("input_133")]; + tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; + tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_35, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; tensor prev_kv_7_end_0 = const()[name = tensor("prev_kv_7_end_0"), val = tensor([4, 1, 4, 64, 64])]; tensor prev_kv_7_end_mask_0 = const()[name = tensor("prev_kv_7_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1422,382 +1078,267 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_1562 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_37)[name = tensor("linear_30")]; - tensor var_1563 = const()[name = tensor("op_1563"), val = tensor([1, 4, 4, 64])]; - tensor var_1564 = reshape(shape = var_1563, x = var_1562)[name = tensor("op_1564")]; + tensor var_1079 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_1080 = const()[name = tensor("op_1080"), val = tensor([1, 4, 4, 64])]; + tensor var_1081 = reshape(shape = var_1080, x = var_1079)[name = tensor("op_1081")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1568 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_37)[name = tensor("linear_31")]; - tensor var_1569 = const()[name = tensor("op_1569"), val = tensor(0x1p-3)]; - tensor var_1570 = mul(x = var_1568, y = var_1569)[name = tensor("op_1570")]; - tensor var_1571 = const()[name = tensor("op_1571"), val = tensor([1, 4, 4, 64])]; - tensor var_1572 = reshape(shape = var_1571, x = var_1570)[name = tensor("op_1572")]; + tensor var_1085 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_1086 = const()[name = tensor("op_1086"), val = tensor(0x1p-3)]; + tensor var_1087 = mul(x = var_1085, y = var_1086)[name = tensor("op_1087")]; + tensor var_1088 = const()[name = tensor("op_1088"), val = tensor([1, 4, 4, 64])]; + tensor var_1089 = reshape(shape = var_1088, x = var_1087)[name = tensor("op_1089")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1576 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_37)[name = tensor("linear_32")]; - tensor var_1577 = const()[name = tensor("op_1577"), val = tensor([1, 4, 4, 64])]; - tensor var_1578 = reshape(shape = var_1577, x = var_1576)[name = tensor("op_1578")]; + tensor var_1093 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_1094 = const()[name = tensor("op_1094"), val = tensor([1, 4, 4, 64])]; + tensor var_1095 = reshape(shape = var_1094, x = var_1093)[name = tensor("op_1095")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_227 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_37)[name = tensor("linear_33")]; + tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; tensor s_t_7 = add(x = prev_scale_7, y = encoder__t_index)[name = tensor("s_t_7")]; tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_1572)[name = tensor("transpose_41")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_1564)[name = tensor("transpose_42")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_1089)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_1081)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_1588 = const()[name = tensor("op_1588"), val = tensor([4, 1])]; - tensor var_1589 = reshape(shape = var_1588, x = sqrt_s_t_7)[name = tensor("op_1589")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_1589)[name = tensor("M_7")]; - tensor var_1591 = mul(x = qk_7, y = M_7)[name = tensor("op_1591")]; + tensor var_1105 = const()[name = tensor("op_1105"), val = tensor([4, 1])]; + tensor var_1106 = reshape(shape = var_1105, x = sqrt_s_t_7)[name = tensor("op_1106")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_1106)[name = tensor("M_7")]; + tensor var_1108 = mul(x = qk_7, y = M_7)[name = tensor("op_1108")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_1578)[name = tensor("transpose_40")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1591, y = v_7)[name = tensor("inner_7")]; - tensor var_1593_transpose_x_0 = const()[name = tensor("op_1593_transpose_x_0"), val = tensor(false)]; - tensor var_1593_transpose_y_0 = const()[name = tensor("op_1593_transpose_y_0"), val = tensor(false)]; - tensor var_1593 = matmul(transpose_x = var_1593_transpose_x_0, transpose_y = var_1593_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1593")]; - tensor var_1594 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1594")]; - tensor var_1595 = const()[name = tensor("op_1595"), val = tensor([1, 1, 4, 1])]; - tensor var_1596 = reshape(shape = var_1595, x = var_1594)[name = tensor("op_1596")]; - tensor cross_7 = mul(x = var_1593, y = var_1596)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_1095)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1108, y = v_7)[name = tensor("inner_7")]; + tensor var_1110_transpose_x_0 = const()[name = tensor("op_1110_transpose_x_0"), val = tensor(false)]; + tensor var_1110_transpose_y_0 = const()[name = tensor("op_1110_transpose_y_0"), val = tensor(false)]; + tensor var_1110 = matmul(transpose_x = var_1110_transpose_x_0, transpose_y = var_1110_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1110")]; + tensor var_1111 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1111")]; + tensor var_1112 = const()[name = tensor("op_1112"), val = tensor([1, 1, 4, 1])]; + tensor var_1113 = reshape(shape = var_1112, x = var_1111)[name = tensor("op_1113")]; + tensor cross_7 = mul(x = var_1110, y = var_1113)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_1599 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1599")]; - tensor var_1601_transpose_x_1 = const()[name = tensor("op_1601_transpose_x_1"), val = tensor(true)]; - tensor var_1601_transpose_y_1 = const()[name = tensor("op_1601_transpose_y_1"), val = tensor(false)]; - tensor var_1601 = matmul(transpose_x = var_1601_transpose_x_1, transpose_y = var_1601_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1601")]; - tensor new_kv_unnorm_7 = add(x = var_1599, y = var_1601)[name = tensor("new_kv_unnorm_7")]; - tensor var_1603 = const()[name = tensor("op_1603"), val = tensor(0x1p+2)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_1603)[name = tensor("new_scale_7")]; - tensor var_1605 = sqrt(x = new_scale_7)[name = tensor("op_1605")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1605)[name = tensor("nkv_1")]; - tensor var_1607_perm_0 = const()[name = tensor("op_1607_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1116 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1116")]; + tensor var_1118_transpose_x_1 = const()[name = tensor("op_1118_transpose_x_1"), val = tensor(true)]; + tensor var_1118_transpose_y_1 = const()[name = tensor("op_1118_transpose_y_1"), val = tensor(false)]; + tensor var_1118 = matmul(transpose_x = var_1118_transpose_x_1, transpose_y = var_1118_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1118")]; + tensor new_kv_unnorm_7 = add(x = var_1116, y = var_1118)[name = tensor("new_kv_unnorm_7")]; + tensor var_1120 = const()[name = tensor("op_1120"), val = tensor(0x1p+2)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_1120)[name = tensor("new_scale_7")]; + tensor var_1122 = sqrt(x = new_scale_7)[name = tensor("op_1122")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1122)[name = tensor("nkv_1")]; + tensor var_1124_perm_0 = const()[name = tensor("op_1124_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_1607 = transpose(perm = var_1607_perm_0, x = out_19)[name = tensor("transpose_39")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1607)[name = tensor("out_21")]; - tensor var_1611 = const()[name = tensor("op_1611"), val = tensor([1, 4, 256])]; - tensor out_23 = reshape(shape = var_1611, x = out_21)[name = tensor("out_23")]; - tensor var_1613 = silu(x = input_227)[name = tensor("op_1613")]; - tensor input_229 = mul(x = var_1613, y = out_23)[name = tensor("input_229")]; - tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_229)[name = tensor("linear_34")]; - tensor x_39 = add(x = input_223, y = ret_out_7)[name = tensor("x_39")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor var_1124 = transpose(perm = var_1124_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_24, x = var_1124)[name = tensor("out_21")]; + tensor var_1128 = const()[name = tensor("op_1128"), val = tensor([1, 4, 256])]; + tensor out_23 = reshape(shape = var_1128, x = out_21)[name = tensor("out_23")]; + tensor var_1130 = silu(x = input_137)[name = tensor("op_1130")]; + tensor input_139 = mul(x = var_1130, y = out_23)[name = tensor("input_139")]; + tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; + tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; + tensor window_31_begin_0 = const()[name = tensor("window_31_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_31_end_0 = const()[name = tensor("window_31_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_31_end_mask_0 = const()[name = tensor("window_31_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_31_squeeze_mask_0 = const()[name = tensor("window_31_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_31 = slice_by_index(begin = window_31_begin_0, end = window_31_end_0, end_mask = window_31_end_mask_0, squeeze_mask = window_31_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_31")]; tensor filled_25_begin_0 = const()[name = tensor("filled_25_begin_0"), val = tensor([3, 0])]; tensor filled_25_end_0 = const()[name = tensor("filled_25_end_0"), val = tensor([4, 1])]; tensor filled_25_end_mask_0 = const()[name = tensor("filled_25_end_mask_0"), val = tensor([false, true])]; tensor filled_25_squeeze_mask_0 = const()[name = tensor("filled_25_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_25 = slice_by_index(begin = filled_25_begin_0, end = filled_25_end_0, end_mask = filled_25_end_mask_0, squeeze_mask = filled_25_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_25")]; - tensor var_1622_begin_0 = const()[name = tensor("op_1622_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1622_end_0 = const()[name = tensor("op_1622_end_0"), val = tensor([1, 1, 256])]; - tensor var_1622_end_mask_0 = const()[name = tensor("op_1622_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1622 = slice_by_index(begin = var_1622_begin_0, end = var_1622_end_0, end_mask = var_1622_end_mask_0, x = x_39)[name = tensor("op_1622")]; - tensor var_1625_begin_0 = const()[name = tensor("op_1625_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1625_end_0 = const()[name = tensor("op_1625_end_0"), val = tensor([1, 16, 256])]; - tensor var_1625_end_mask_0 = const()[name = tensor("op_1625_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1625 = slice_by_index(begin = var_1625_begin_0, end = var_1625_end_0, end_mask = var_1625_end_mask_0, x = window)[name = tensor("op_1625")]; + tensor var_1139_begin_0 = const()[name = tensor("op_1139_begin_0"), val = tensor([0, 0, 0])]; + tensor var_1139_end_0 = const()[name = tensor("op_1139_end_0"), val = tensor([1, 1, 256])]; + tensor var_1139_end_mask_0 = const()[name = tensor("op_1139_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1139 = slice_by_index(begin = var_1139_begin_0, end = var_1139_end_0, end_mask = var_1139_end_mask_0, x = x_21)[name = tensor("op_1139")]; + tensor var_1142_begin_0 = const()[name = tensor("op_1142_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1142_end_0 = const()[name = tensor("op_1142_end_0"), val = tensor([1, 16, 256])]; + tensor var_1142_end_mask_0 = const()[name = tensor("op_1142_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1142 = slice_by_index(begin = var_1142_begin_0, end = var_1142_end_0, end_mask = var_1142_end_mask_0, x = window_31)[name = tensor("op_1142")]; tensor rolled_25_interleave_0 = const()[name = tensor("rolled_25_interleave_0"), val = tensor(false)]; - tensor rolled_25 = concat(axis = var_33, interleave = rolled_25_interleave_0, values = (var_1625, var_1622))[name = tensor("rolled_25")]; + tensor rolled_25 = concat(axis = var_33, interleave = rolled_25_interleave_0, values = (var_1142, var_1139))[name = tensor("rolled_25")]; tensor clip_24 = clip(alpha = var_21, beta = var_20, x = filled_25)[name = tensor("clip_24")]; - tensor var_1630 = equal(x = encoder__conv_positions, y = clip_24)[name = tensor("op_1630")]; - tensor var_1631_dtype_0 = const()[name = tensor("op_1631_dtype_0"), val = tensor("fp32")]; - tensor var_1632 = const()[name = tensor("op_1632"), val = tensor([1, 16, 1])]; - tensor var_1631 = cast(dtype = var_1631_dtype_0, x = var_1630)[name = tensor("cast_151")]; - tensor fill_mask_25 = reshape(shape = var_1632, x = var_1631)[name = tensor("fill_mask_25")]; + tensor var_1147 = equal(x = encoder__conv_positions, y = clip_24)[name = tensor("op_1147")]; + tensor var_1148_dtype_0 = const()[name = tensor("op_1148_dtype_0"), val = tensor("fp32")]; + tensor var_1149 = const()[name = tensor("op_1149"), val = tensor([1, 16, 1])]; + tensor var_1148 = cast(dtype = var_1148_dtype_0, x = var_1147)[name = tensor("cast_151")]; + tensor fill_mask_25 = reshape(shape = var_1149, x = var_1148)[name = tensor("fill_mask_25")]; tensor x_expanded_25_reps_0 = const()[name = tensor("x_expanded_25_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_25 = tile(reps = x_expanded_25_reps_0, x = var_1622)[name = tensor("x_expanded_25")]; - tensor var_1636 = sub(x = var_17, y = fill_mask_25)[name = tensor("op_1636")]; - tensor var_1637 = mul(x = window, y = var_1636)[name = tensor("op_1637")]; - tensor var_1638 = mul(x = x_expanded_25, y = fill_mask_25)[name = tensor("op_1638")]; - tensor filled_window_25 = add(x = var_1637, y = var_1638)[name = tensor("filled_window_25")]; - tensor var_1640 = greater_equal(x = filled_25, y = var_16)[name = tensor("op_1640")]; - tensor var_1641_dtype_0 = const()[name = tensor("op_1641_dtype_0"), val = tensor("fp32")]; - tensor var_1642 = const()[name = tensor("op_1642"), val = tensor([1, 1, 1])]; - tensor var_1641 = cast(dtype = var_1641_dtype_0, x = var_1640)[name = tensor("cast_150")]; - tensor use_roll_25 = reshape(shape = var_1642, x = var_1641)[name = tensor("use_roll_25")]; - tensor var_1644 = sub(x = var_17, y = use_roll_25)[name = tensor("op_1644")]; - tensor var_1645 = mul(x = filled_window_25, y = var_1644)[name = tensor("op_1645")]; - tensor var_1646 = mul(x = rolled_25, y = use_roll_25)[name = tensor("op_1646")]; - tensor input_231 = add(x = var_1645, y = var_1646)[name = tensor("input_231")]; - tensor var_1648 = const()[name = tensor("op_1648"), val = tensor(0x1p+0)]; - tensor var_1649 = add(x = filled_25, y = var_1648)[name = tensor("op_1649")]; - tensor clip_25 = clip(alpha = var_21, beta = var_16, x = var_1649)[name = tensor("clip_25")]; - tensor x_41_axes_0 = const()[name = tensor("x_41_axes_0"), val = tensor([-1])]; - tensor x_41 = layer_norm(axes = x_41_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_231)[name = tensor("x_41")]; - tensor input_233_perm_0 = const()[name = tensor("input_233_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_87_pad_type_0 = const()[name = tensor("inputs_87_pad_type_0"), val = tensor("valid")]; - tensor inputs_87_strides_0 = const()[name = tensor("inputs_87_strides_0"), val = tensor([1])]; - tensor inputs_87_pad_0 = const()[name = tensor("inputs_87_pad_0"), val = tensor([0, 0])]; - tensor inputs_87_dilations_0 = const()[name = tensor("inputs_87_dilations_0"), val = tensor([1])]; - tensor inputs_87_groups_0 = const()[name = tensor("inputs_87_groups_0"), val = tensor(1)]; - tensor input_233 = transpose(perm = input_233_perm_0, x = x_41)[name = tensor("transpose_38")]; - tensor inputs_87 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_87_dilations_0, groups = inputs_87_groups_0, pad = inputs_87_pad_0, pad_type = inputs_87_pad_type_0, strides = inputs_87_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_233)[name = tensor("inputs_87")]; - tensor var_1670_split_sizes_0 = const()[name = tensor("op_1670_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1670_axis_0 = const()[name = tensor("op_1670_axis_0"), val = tensor(1)]; - tensor var_1670_0, tensor var_1670_1 = split(axis = var_1670_axis_0, split_sizes = var_1670_split_sizes_0, x = inputs_87)[name = tensor("op_1670")]; - tensor var_1672 = sigmoid(x = var_1670_1)[name = tensor("op_1672")]; - tensor inputs_89 = mul(x = var_1670_0, y = var_1672)[name = tensor("inputs_89")]; - tensor outputs_aug_25_pad_type_0 = const()[name = tensor("outputs_aug_25_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_25_pad_0 = const()[name = tensor("outputs_aug_25_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_25_groups_0 = const()[name = tensor("outputs_aug_25_groups_0"), val = tensor(256)]; - tensor outputs_aug_25_strides_0 = const()[name = tensor("outputs_aug_25_strides_0"), val = tensor([1])]; - tensor outputs_aug_25_dilations_0 = const()[name = tensor("outputs_aug_25_dilations_0"), val = tensor([1])]; - tensor outputs_aug_25 = conv(dilations = outputs_aug_25_dilations_0, groups = outputs_aug_25_groups_0, pad = outputs_aug_25_pad_0, pad_type = outputs_aug_25_pad_type_0, strides = outputs_aug_25_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_89)[name = tensor("outputs_aug_25")]; - tensor input_235_begin_0 = const()[name = tensor("input_235_begin_0"), val = tensor([0, 0, 0])]; - tensor input_235_end_0 = const()[name = tensor("input_235_end_0"), val = tensor([1, 256, 16])]; - tensor input_235_end_mask_0 = const()[name = tensor("input_235_end_mask_0"), val = tensor([true, true, false])]; - tensor input_235 = slice_by_index(begin = input_235_begin_0, end = input_235_end_0, end_mask = input_235_end_mask_0, x = outputs_aug_25)[name = tensor("input_235")]; - tensor inputs_91 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_235)[name = tensor("inputs_91")]; - tensor input_237 = silu(x = inputs_91)[name = tensor("input_237")]; - tensor input_239_pad_type_0 = const()[name = tensor("input_239_pad_type_0"), val = tensor("valid")]; - tensor input_239_strides_0 = const()[name = tensor("input_239_strides_0"), val = tensor([1])]; - tensor input_239_pad_0 = const()[name = tensor("input_239_pad_0"), val = tensor([0, 0])]; - tensor input_239_dilations_0 = const()[name = tensor("input_239_dilations_0"), val = tensor([1])]; - tensor input_239_groups_0 = const()[name = tensor("input_239_groups_0"), val = tensor(1)]; - tensor input_239 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_239_dilations_0, groups = input_239_groups_0, pad = input_239_pad_0, pad_type = input_239_pad_type_0, strides = input_239_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_237)[name = tensor("input_239")]; - tensor conv_out_25_perm_0 = const()[name = tensor("conv_out_25_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1703_begin_0 = const()[name = tensor("op_1703_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1703_end_0 = const()[name = tensor("op_1703_end_0"), val = tensor([1, 16, 256])]; - tensor var_1703_end_mask_0 = const()[name = tensor("op_1703_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_25 = transpose(perm = conv_out_25_perm_0, x = input_239)[name = tensor("transpose_37")]; - tensor var_1703 = slice_by_index(begin = var_1703_begin_0, end = var_1703_end_0, end_mask = var_1703_end_mask_0, x = conv_out_25)[name = tensor("op_1703")]; - tensor var_1706_begin_0 = const()[name = tensor("op_1706_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1706_end_0 = const()[name = tensor("op_1706_end_0"), val = tensor([1, 2, 256])]; - tensor var_1706_end_mask_0 = const()[name = tensor("op_1706_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1706 = slice_by_index(begin = var_1706_begin_0, end = var_1706_end_0, end_mask = var_1706_end_mask_0, x = x_39)[name = tensor("op_1706")]; - tensor var_1709_begin_0 = const()[name = tensor("op_1709_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1709_end_0 = const()[name = tensor("op_1709_end_0"), val = tensor([1, 16, 256])]; - tensor var_1709_end_mask_0 = const()[name = tensor("op_1709_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1709 = slice_by_index(begin = var_1709_begin_0, end = var_1709_end_0, end_mask = var_1709_end_mask_0, x = input_231)[name = tensor("op_1709")]; + tensor x_expanded_25 = tile(reps = x_expanded_25_reps_0, x = var_1139)[name = tensor("x_expanded_25")]; + tensor var_1153 = sub(x = var_17, y = fill_mask_25)[name = tensor("op_1153")]; + tensor var_1154 = mul(x = window_31, y = var_1153)[name = tensor("op_1154")]; + tensor var_1155 = mul(x = x_expanded_25, y = fill_mask_25)[name = tensor("op_1155")]; + tensor filled_window_25 = add(x = var_1154, y = var_1155)[name = tensor("filled_window_25")]; + tensor var_1157 = greater_equal(x = filled_25, y = var_16)[name = tensor("op_1157")]; + tensor var_1158_dtype_0 = const()[name = tensor("op_1158_dtype_0"), val = tensor("fp32")]; + tensor var_1159 = const()[name = tensor("op_1159"), val = tensor([1, 1, 1])]; + tensor var_1158 = cast(dtype = var_1158_dtype_0, x = var_1157)[name = tensor("cast_150")]; + tensor use_roll_25 = reshape(shape = var_1159, x = var_1158)[name = tensor("use_roll_25")]; + tensor var_1161 = sub(x = var_17, y = use_roll_25)[name = tensor("op_1161")]; + tensor var_1162 = mul(x = filled_window_25, y = var_1161)[name = tensor("op_1162")]; + tensor var_1163 = mul(x = rolled_25, y = use_roll_25)[name = tensor("op_1163")]; + tensor window_33 = add(x = var_1162, y = var_1163)[name = tensor("window_33")]; + tensor var_1165 = const()[name = tensor("op_1165"), val = tensor(0x1p+0)]; + tensor var_1166 = add(x = filled_25, y = var_1165)[name = tensor("op_1166")]; + tensor clip_25 = clip(alpha = var_21, beta = var_16, x = var_1166)[name = tensor("clip_25")]; + tensor var_1169_begin_0 = const()[name = tensor("op_1169_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1169_end_0 = const()[name = tensor("op_1169_end_0"), val = tensor([1, 2, 256])]; + tensor var_1169_end_mask_0 = const()[name = tensor("op_1169_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1169 = slice_by_index(begin = var_1169_begin_0, end = var_1169_end_0, end_mask = var_1169_end_mask_0, x = x_21)[name = tensor("op_1169")]; + tensor var_1172_begin_0 = const()[name = tensor("op_1172_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1172_end_0 = const()[name = tensor("op_1172_end_0"), val = tensor([1, 16, 256])]; + tensor var_1172_end_mask_0 = const()[name = tensor("op_1172_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1172 = slice_by_index(begin = var_1172_begin_0, end = var_1172_end_0, end_mask = var_1172_end_mask_0, x = window_33)[name = tensor("op_1172")]; tensor rolled_27_interleave_0 = const()[name = tensor("rolled_27_interleave_0"), val = tensor(false)]; - tensor rolled_27 = concat(axis = var_33, interleave = rolled_27_interleave_0, values = (var_1709, var_1706))[name = tensor("rolled_27")]; + tensor rolled_27 = concat(axis = var_33, interleave = rolled_27_interleave_0, values = (var_1172, var_1169))[name = tensor("rolled_27")]; tensor clip_26 = clip(alpha = var_21, beta = var_20, x = clip_25)[name = tensor("clip_26")]; - tensor var_1714 = equal(x = encoder__conv_positions, y = clip_26)[name = tensor("op_1714")]; - tensor var_1715_dtype_0 = const()[name = tensor("op_1715_dtype_0"), val = tensor("fp32")]; - tensor var_1716 = const()[name = tensor("op_1716"), val = tensor([1, 16, 1])]; - tensor var_1715 = cast(dtype = var_1715_dtype_0, x = var_1714)[name = tensor("cast_149")]; - tensor fill_mask_27 = reshape(shape = var_1716, x = var_1715)[name = tensor("fill_mask_27")]; + tensor var_1177 = equal(x = encoder__conv_positions, y = clip_26)[name = tensor("op_1177")]; + tensor var_1178_dtype_0 = const()[name = tensor("op_1178_dtype_0"), val = tensor("fp32")]; + tensor var_1179 = const()[name = tensor("op_1179"), val = tensor([1, 16, 1])]; + tensor var_1178 = cast(dtype = var_1178_dtype_0, x = var_1177)[name = tensor("cast_149")]; + tensor fill_mask_27 = reshape(shape = var_1179, x = var_1178)[name = tensor("fill_mask_27")]; tensor x_expanded_27_reps_0 = const()[name = tensor("x_expanded_27_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_27 = tile(reps = x_expanded_27_reps_0, x = var_1706)[name = tensor("x_expanded_27")]; - tensor var_1720 = sub(x = var_17, y = fill_mask_27)[name = tensor("op_1720")]; - tensor var_1721 = mul(x = input_231, y = var_1720)[name = tensor("op_1721")]; - tensor var_1722 = mul(x = x_expanded_27, y = fill_mask_27)[name = tensor("op_1722")]; - tensor filled_window_27 = add(x = var_1721, y = var_1722)[name = tensor("filled_window_27")]; - tensor var_1724 = greater_equal(x = clip_25, y = var_16)[name = tensor("op_1724")]; - tensor var_1725_dtype_0 = const()[name = tensor("op_1725_dtype_0"), val = tensor("fp32")]; - tensor var_1726 = const()[name = tensor("op_1726"), val = tensor([1, 1, 1])]; - tensor var_1725 = cast(dtype = var_1725_dtype_0, x = var_1724)[name = tensor("cast_148")]; - tensor use_roll_27 = reshape(shape = var_1726, x = var_1725)[name = tensor("use_roll_27")]; - tensor var_1728 = sub(x = var_17, y = use_roll_27)[name = tensor("op_1728")]; - tensor var_1729 = mul(x = filled_window_27, y = var_1728)[name = tensor("op_1729")]; - tensor var_1730 = mul(x = rolled_27, y = use_roll_27)[name = tensor("op_1730")]; - tensor input_241 = add(x = var_1729, y = var_1730)[name = tensor("input_241")]; - tensor var_1732 = const()[name = tensor("op_1732"), val = tensor(0x1p+0)]; - tensor var_1733 = add(x = clip_25, y = var_1732)[name = tensor("op_1733")]; - tensor clip_27 = clip(alpha = var_21, beta = var_16, x = var_1733)[name = tensor("clip_27")]; - tensor x_43_axes_0 = const()[name = tensor("x_43_axes_0"), val = tensor([-1])]; - tensor x_43 = layer_norm(axes = x_43_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_241)[name = tensor("x_43")]; - tensor input_243_perm_0 = const()[name = tensor("input_243_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_93_pad_type_0 = const()[name = tensor("inputs_93_pad_type_0"), val = tensor("valid")]; - tensor inputs_93_strides_0 = const()[name = tensor("inputs_93_strides_0"), val = tensor([1])]; - tensor inputs_93_pad_0 = const()[name = tensor("inputs_93_pad_0"), val = tensor([0, 0])]; - tensor inputs_93_dilations_0 = const()[name = tensor("inputs_93_dilations_0"), val = tensor([1])]; - tensor inputs_93_groups_0 = const()[name = tensor("inputs_93_groups_0"), val = tensor(1)]; - tensor input_243 = transpose(perm = input_243_perm_0, x = x_43)[name = tensor("transpose_36")]; - tensor inputs_93 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_93_dilations_0, groups = inputs_93_groups_0, pad = inputs_93_pad_0, pad_type = inputs_93_pad_type_0, strides = inputs_93_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_243)[name = tensor("inputs_93")]; - tensor var_1754_split_sizes_0 = const()[name = tensor("op_1754_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1754_axis_0 = const()[name = tensor("op_1754_axis_0"), val = tensor(1)]; - tensor var_1754_0, tensor var_1754_1 = split(axis = var_1754_axis_0, split_sizes = var_1754_split_sizes_0, x = inputs_93)[name = tensor("op_1754")]; - tensor var_1756 = sigmoid(x = var_1754_1)[name = tensor("op_1756")]; - tensor inputs_95 = mul(x = var_1754_0, y = var_1756)[name = tensor("inputs_95")]; - tensor outputs_aug_27_pad_type_0 = const()[name = tensor("outputs_aug_27_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_27_pad_0 = const()[name = tensor("outputs_aug_27_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_27_groups_0 = const()[name = tensor("outputs_aug_27_groups_0"), val = tensor(256)]; - tensor outputs_aug_27_strides_0 = const()[name = tensor("outputs_aug_27_strides_0"), val = tensor([1])]; - tensor outputs_aug_27_dilations_0 = const()[name = tensor("outputs_aug_27_dilations_0"), val = tensor([1])]; - tensor outputs_aug_27 = conv(dilations = outputs_aug_27_dilations_0, groups = outputs_aug_27_groups_0, pad = outputs_aug_27_pad_0, pad_type = outputs_aug_27_pad_type_0, strides = outputs_aug_27_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_95)[name = tensor("outputs_aug_27")]; - tensor input_245_begin_0 = const()[name = tensor("input_245_begin_0"), val = tensor([0, 0, 0])]; - tensor input_245_end_0 = const()[name = tensor("input_245_end_0"), val = tensor([1, 256, 16])]; - tensor input_245_end_mask_0 = const()[name = tensor("input_245_end_mask_0"), val = tensor([true, true, false])]; - tensor input_245 = slice_by_index(begin = input_245_begin_0, end = input_245_end_0, end_mask = input_245_end_mask_0, x = outputs_aug_27)[name = tensor("input_245")]; - tensor inputs_97 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_245)[name = tensor("inputs_97")]; - tensor input_247 = silu(x = inputs_97)[name = tensor("input_247")]; - tensor input_249_pad_type_0 = const()[name = tensor("input_249_pad_type_0"), val = tensor("valid")]; - tensor input_249_strides_0 = const()[name = tensor("input_249_strides_0"), val = tensor([1])]; - tensor input_249_pad_0 = const()[name = tensor("input_249_pad_0"), val = tensor([0, 0])]; - tensor input_249_dilations_0 = const()[name = tensor("input_249_dilations_0"), val = tensor([1])]; - tensor input_249_groups_0 = const()[name = tensor("input_249_groups_0"), val = tensor(1)]; - tensor input_249 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_249_dilations_0, groups = input_249_groups_0, pad = input_249_pad_0, pad_type = input_249_pad_type_0, strides = input_249_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_247)[name = tensor("input_249")]; - tensor conv_out_27_perm_0 = const()[name = tensor("conv_out_27_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1787_begin_0 = const()[name = tensor("op_1787_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1787_end_0 = const()[name = tensor("op_1787_end_0"), val = tensor([1, 16, 256])]; - tensor var_1787_end_mask_0 = const()[name = tensor("op_1787_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_27 = transpose(perm = conv_out_27_perm_0, x = input_249)[name = tensor("transpose_35")]; - tensor var_1787 = slice_by_index(begin = var_1787_begin_0, end = var_1787_end_0, end_mask = var_1787_end_mask_0, x = conv_out_27)[name = tensor("op_1787")]; - tensor var_1790_begin_0 = const()[name = tensor("op_1790_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1790_end_0 = const()[name = tensor("op_1790_end_0"), val = tensor([1, 3, 256])]; - tensor var_1790_end_mask_0 = const()[name = tensor("op_1790_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1790 = slice_by_index(begin = var_1790_begin_0, end = var_1790_end_0, end_mask = var_1790_end_mask_0, x = x_39)[name = tensor("op_1790")]; - tensor var_1793_begin_0 = const()[name = tensor("op_1793_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1793_end_0 = const()[name = tensor("op_1793_end_0"), val = tensor([1, 16, 256])]; - tensor var_1793_end_mask_0 = const()[name = tensor("op_1793_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1793 = slice_by_index(begin = var_1793_begin_0, end = var_1793_end_0, end_mask = var_1793_end_mask_0, x = input_241)[name = tensor("op_1793")]; + tensor x_expanded_27 = tile(reps = x_expanded_27_reps_0, x = var_1169)[name = tensor("x_expanded_27")]; + tensor var_1183 = sub(x = var_17, y = fill_mask_27)[name = tensor("op_1183")]; + tensor var_1184 = mul(x = window_33, y = var_1183)[name = tensor("op_1184")]; + tensor var_1185 = mul(x = x_expanded_27, y = fill_mask_27)[name = tensor("op_1185")]; + tensor filled_window_27 = add(x = var_1184, y = var_1185)[name = tensor("filled_window_27")]; + tensor var_1187 = greater_equal(x = clip_25, y = var_16)[name = tensor("op_1187")]; + tensor var_1188_dtype_0 = const()[name = tensor("op_1188_dtype_0"), val = tensor("fp32")]; + tensor var_1189 = const()[name = tensor("op_1189"), val = tensor([1, 1, 1])]; + tensor var_1188 = cast(dtype = var_1188_dtype_0, x = var_1187)[name = tensor("cast_148")]; + tensor use_roll_27 = reshape(shape = var_1189, x = var_1188)[name = tensor("use_roll_27")]; + tensor var_1191 = sub(x = var_17, y = use_roll_27)[name = tensor("op_1191")]; + tensor var_1192 = mul(x = filled_window_27, y = var_1191)[name = tensor("op_1192")]; + tensor var_1193 = mul(x = rolled_27, y = use_roll_27)[name = tensor("op_1193")]; + tensor window_35 = add(x = var_1192, y = var_1193)[name = tensor("window_35")]; + tensor var_1195 = const()[name = tensor("op_1195"), val = tensor(0x1p+0)]; + tensor var_1196 = add(x = clip_25, y = var_1195)[name = tensor("op_1196")]; + tensor clip_27 = clip(alpha = var_21, beta = var_16, x = var_1196)[name = tensor("clip_27")]; + tensor var_1199_begin_0 = const()[name = tensor("op_1199_begin_0"), val = tensor([0, 2, 0])]; + tensor var_1199_end_0 = const()[name = tensor("op_1199_end_0"), val = tensor([1, 3, 256])]; + tensor var_1199_end_mask_0 = const()[name = tensor("op_1199_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1199 = slice_by_index(begin = var_1199_begin_0, end = var_1199_end_0, end_mask = var_1199_end_mask_0, x = x_21)[name = tensor("op_1199")]; + tensor var_1202_begin_0 = const()[name = tensor("op_1202_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1202_end_0 = const()[name = tensor("op_1202_end_0"), val = tensor([1, 16, 256])]; + tensor var_1202_end_mask_0 = const()[name = tensor("op_1202_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1202 = slice_by_index(begin = var_1202_begin_0, end = var_1202_end_0, end_mask = var_1202_end_mask_0, x = window_35)[name = tensor("op_1202")]; tensor rolled_29_interleave_0 = const()[name = tensor("rolled_29_interleave_0"), val = tensor(false)]; - tensor rolled_29 = concat(axis = var_33, interleave = rolled_29_interleave_0, values = (var_1793, var_1790))[name = tensor("rolled_29")]; + tensor rolled_29 = concat(axis = var_33, interleave = rolled_29_interleave_0, values = (var_1202, var_1199))[name = tensor("rolled_29")]; tensor clip_28 = clip(alpha = var_21, beta = var_20, x = clip_27)[name = tensor("clip_28")]; - tensor var_1798 = equal(x = encoder__conv_positions, y = clip_28)[name = tensor("op_1798")]; - tensor var_1799_dtype_0 = const()[name = tensor("op_1799_dtype_0"), val = tensor("fp32")]; - tensor var_1800 = const()[name = tensor("op_1800"), val = tensor([1, 16, 1])]; - tensor var_1799 = cast(dtype = var_1799_dtype_0, x = var_1798)[name = tensor("cast_147")]; - tensor fill_mask_29 = reshape(shape = var_1800, x = var_1799)[name = tensor("fill_mask_29")]; + tensor var_1207 = equal(x = encoder__conv_positions, y = clip_28)[name = tensor("op_1207")]; + tensor var_1208_dtype_0 = const()[name = tensor("op_1208_dtype_0"), val = tensor("fp32")]; + tensor var_1209 = const()[name = tensor("op_1209"), val = tensor([1, 16, 1])]; + tensor var_1208 = cast(dtype = var_1208_dtype_0, x = var_1207)[name = tensor("cast_147")]; + tensor fill_mask_29 = reshape(shape = var_1209, x = var_1208)[name = tensor("fill_mask_29")]; tensor x_expanded_29_reps_0 = const()[name = tensor("x_expanded_29_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_29 = tile(reps = x_expanded_29_reps_0, x = var_1790)[name = tensor("x_expanded_29")]; - tensor var_1804 = sub(x = var_17, y = fill_mask_29)[name = tensor("op_1804")]; - tensor var_1805 = mul(x = input_241, y = var_1804)[name = tensor("op_1805")]; - tensor var_1806 = mul(x = x_expanded_29, y = fill_mask_29)[name = tensor("op_1806")]; - tensor filled_window_29 = add(x = var_1805, y = var_1806)[name = tensor("filled_window_29")]; - tensor var_1808 = greater_equal(x = clip_27, y = var_16)[name = tensor("op_1808")]; - tensor var_1809_dtype_0 = const()[name = tensor("op_1809_dtype_0"), val = tensor("fp32")]; - tensor var_1810 = const()[name = tensor("op_1810"), val = tensor([1, 1, 1])]; - tensor var_1809 = cast(dtype = var_1809_dtype_0, x = var_1808)[name = tensor("cast_146")]; - tensor use_roll_29 = reshape(shape = var_1810, x = var_1809)[name = tensor("use_roll_29")]; - tensor var_1812 = sub(x = var_17, y = use_roll_29)[name = tensor("op_1812")]; - tensor var_1813 = mul(x = filled_window_29, y = var_1812)[name = tensor("op_1813")]; - tensor var_1814 = mul(x = rolled_29, y = use_roll_29)[name = tensor("op_1814")]; - tensor input_251 = add(x = var_1813, y = var_1814)[name = tensor("input_251")]; - tensor var_1816 = const()[name = tensor("op_1816"), val = tensor(0x1p+0)]; - tensor var_1817 = add(x = clip_27, y = var_1816)[name = tensor("op_1817")]; - tensor clip_29 = clip(alpha = var_21, beta = var_16, x = var_1817)[name = tensor("clip_29")]; - tensor x_45_axes_0 = const()[name = tensor("x_45_axes_0"), val = tensor([-1])]; - tensor x_45 = layer_norm(axes = x_45_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_251)[name = tensor("x_45")]; - tensor input_253_perm_0 = const()[name = tensor("input_253_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_99_pad_type_0 = const()[name = tensor("inputs_99_pad_type_0"), val = tensor("valid")]; - tensor inputs_99_strides_0 = const()[name = tensor("inputs_99_strides_0"), val = tensor([1])]; - tensor inputs_99_pad_0 = const()[name = tensor("inputs_99_pad_0"), val = tensor([0, 0])]; - tensor inputs_99_dilations_0 = const()[name = tensor("inputs_99_dilations_0"), val = tensor([1])]; - tensor inputs_99_groups_0 = const()[name = tensor("inputs_99_groups_0"), val = tensor(1)]; - tensor input_253 = transpose(perm = input_253_perm_0, x = x_45)[name = tensor("transpose_34")]; - tensor inputs_99 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_99_dilations_0, groups = inputs_99_groups_0, pad = inputs_99_pad_0, pad_type = inputs_99_pad_type_0, strides = inputs_99_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_253)[name = tensor("inputs_99")]; - tensor var_1838_split_sizes_0 = const()[name = tensor("op_1838_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1838_axis_0 = const()[name = tensor("op_1838_axis_0"), val = tensor(1)]; - tensor var_1838_0, tensor var_1838_1 = split(axis = var_1838_axis_0, split_sizes = var_1838_split_sizes_0, x = inputs_99)[name = tensor("op_1838")]; - tensor var_1840 = sigmoid(x = var_1838_1)[name = tensor("op_1840")]; - tensor inputs_101 = mul(x = var_1838_0, y = var_1840)[name = tensor("inputs_101")]; - tensor outputs_aug_29_pad_type_0 = const()[name = tensor("outputs_aug_29_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_29_pad_0 = const()[name = tensor("outputs_aug_29_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_29_groups_0 = const()[name = tensor("outputs_aug_29_groups_0"), val = tensor(256)]; - tensor outputs_aug_29_strides_0 = const()[name = tensor("outputs_aug_29_strides_0"), val = tensor([1])]; - tensor outputs_aug_29_dilations_0 = const()[name = tensor("outputs_aug_29_dilations_0"), val = tensor([1])]; - tensor outputs_aug_29 = conv(dilations = outputs_aug_29_dilations_0, groups = outputs_aug_29_groups_0, pad = outputs_aug_29_pad_0, pad_type = outputs_aug_29_pad_type_0, strides = outputs_aug_29_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_101)[name = tensor("outputs_aug_29")]; - tensor input_255_begin_0 = const()[name = tensor("input_255_begin_0"), val = tensor([0, 0, 0])]; - tensor input_255_end_0 = const()[name = tensor("input_255_end_0"), val = tensor([1, 256, 16])]; - tensor input_255_end_mask_0 = const()[name = tensor("input_255_end_mask_0"), val = tensor([true, true, false])]; - tensor input_255 = slice_by_index(begin = input_255_begin_0, end = input_255_end_0, end_mask = input_255_end_mask_0, x = outputs_aug_29)[name = tensor("input_255")]; - tensor inputs_103 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_255)[name = tensor("inputs_103")]; - tensor input_257 = silu(x = inputs_103)[name = tensor("input_257")]; - tensor input_259_pad_type_0 = const()[name = tensor("input_259_pad_type_0"), val = tensor("valid")]; - tensor input_259_strides_0 = const()[name = tensor("input_259_strides_0"), val = tensor([1])]; - tensor input_259_pad_0 = const()[name = tensor("input_259_pad_0"), val = tensor([0, 0])]; - tensor input_259_dilations_0 = const()[name = tensor("input_259_dilations_0"), val = tensor([1])]; - tensor input_259_groups_0 = const()[name = tensor("input_259_groups_0"), val = tensor(1)]; - tensor input_259 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_259_dilations_0, groups = input_259_groups_0, pad = input_259_pad_0, pad_type = input_259_pad_type_0, strides = input_259_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_257)[name = tensor("input_259")]; - tensor conv_out_29_perm_0 = const()[name = tensor("conv_out_29_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1871_begin_0 = const()[name = tensor("op_1871_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1871_end_0 = const()[name = tensor("op_1871_end_0"), val = tensor([1, 16, 256])]; - tensor var_1871_end_mask_0 = const()[name = tensor("op_1871_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_29 = transpose(perm = conv_out_29_perm_0, x = input_259)[name = tensor("transpose_33")]; - tensor var_1871 = slice_by_index(begin = var_1871_begin_0, end = var_1871_end_0, end_mask = var_1871_end_mask_0, x = conv_out_29)[name = tensor("op_1871")]; - tensor var_1874_begin_0 = const()[name = tensor("op_1874_begin_0"), val = tensor([0, 3, 0])]; - tensor var_1874_end_0 = const()[name = tensor("op_1874_end_0"), val = tensor([1, 1, 256])]; - tensor var_1874_end_mask_0 = const()[name = tensor("op_1874_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1874 = slice_by_index(begin = var_1874_begin_0, end = var_1874_end_0, end_mask = var_1874_end_mask_0, x = x_39)[name = tensor("op_1874")]; - tensor var_1877_begin_0 = const()[name = tensor("op_1877_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1877_end_0 = const()[name = tensor("op_1877_end_0"), val = tensor([1, 16, 256])]; - tensor var_1877_end_mask_0 = const()[name = tensor("op_1877_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1877 = slice_by_index(begin = var_1877_begin_0, end = var_1877_end_0, end_mask = var_1877_end_mask_0, x = input_251)[name = tensor("op_1877")]; + tensor x_expanded_29 = tile(reps = x_expanded_29_reps_0, x = var_1199)[name = tensor("x_expanded_29")]; + tensor var_1213 = sub(x = var_17, y = fill_mask_29)[name = tensor("op_1213")]; + tensor var_1214 = mul(x = window_35, y = var_1213)[name = tensor("op_1214")]; + tensor var_1215 = mul(x = x_expanded_29, y = fill_mask_29)[name = tensor("op_1215")]; + tensor filled_window_29 = add(x = var_1214, y = var_1215)[name = tensor("filled_window_29")]; + tensor var_1217 = greater_equal(x = clip_27, y = var_16)[name = tensor("op_1217")]; + tensor var_1218_dtype_0 = const()[name = tensor("op_1218_dtype_0"), val = tensor("fp32")]; + tensor var_1219 = const()[name = tensor("op_1219"), val = tensor([1, 1, 1])]; + tensor var_1218 = cast(dtype = var_1218_dtype_0, x = var_1217)[name = tensor("cast_146")]; + tensor use_roll_29 = reshape(shape = var_1219, x = var_1218)[name = tensor("use_roll_29")]; + tensor var_1221 = sub(x = var_17, y = use_roll_29)[name = tensor("op_1221")]; + tensor var_1222 = mul(x = filled_window_29, y = var_1221)[name = tensor("op_1222")]; + tensor var_1223 = mul(x = rolled_29, y = use_roll_29)[name = tensor("op_1223")]; + tensor window_37 = add(x = var_1222, y = var_1223)[name = tensor("window_37")]; + tensor var_1225 = const()[name = tensor("op_1225"), val = tensor(0x1p+0)]; + tensor var_1226 = add(x = clip_27, y = var_1225)[name = tensor("op_1226")]; + tensor clip_29 = clip(alpha = var_21, beta = var_16, x = var_1226)[name = tensor("clip_29")]; + tensor var_1229_begin_0 = const()[name = tensor("op_1229_begin_0"), val = tensor([0, 3, 0])]; + tensor var_1229_end_0 = const()[name = tensor("op_1229_end_0"), val = tensor([1, 1, 256])]; + tensor var_1229_end_mask_0 = const()[name = tensor("op_1229_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1229 = slice_by_index(begin = var_1229_begin_0, end = var_1229_end_0, end_mask = var_1229_end_mask_0, x = x_21)[name = tensor("op_1229")]; + tensor var_1232_begin_0 = const()[name = tensor("op_1232_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1232_end_0 = const()[name = tensor("op_1232_end_0"), val = tensor([1, 16, 256])]; + tensor var_1232_end_mask_0 = const()[name = tensor("op_1232_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1232 = slice_by_index(begin = var_1232_begin_0, end = var_1232_end_0, end_mask = var_1232_end_mask_0, x = window_37)[name = tensor("op_1232")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1877, var_1874))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_33, interleave = rolled_interleave_0, values = (var_1232, var_1229))[name = tensor("rolled")]; tensor clip_30 = clip(alpha = var_21, beta = var_20, x = clip_29)[name = tensor("clip_30")]; - tensor var_1882 = equal(x = encoder__conv_positions, y = clip_30)[name = tensor("op_1882")]; - tensor var_1883_dtype_0 = const()[name = tensor("op_1883_dtype_0"), val = tensor("fp32")]; - tensor var_1884 = const()[name = tensor("op_1884"), val = tensor([1, 16, 1])]; - tensor var_1883 = cast(dtype = var_1883_dtype_0, x = var_1882)[name = tensor("cast_145")]; - tensor fill_mask = reshape(shape = var_1884, x = var_1883)[name = tensor("fill_mask")]; + tensor var_1237 = equal(x = encoder__conv_positions, y = clip_30)[name = tensor("op_1237")]; + tensor var_1238_dtype_0 = const()[name = tensor("op_1238_dtype_0"), val = tensor("fp32")]; + tensor var_1239 = const()[name = tensor("op_1239"), val = tensor([1, 16, 1])]; + tensor var_1238 = cast(dtype = var_1238_dtype_0, x = var_1237)[name = tensor("cast_145")]; + tensor fill_mask = reshape(shape = var_1239, x = var_1238)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1874)[name = tensor("x_expanded")]; - tensor var_1888 = sub(x = var_17, y = fill_mask)[name = tensor("op_1888")]; - tensor var_1889 = mul(x = input_251, y = var_1888)[name = tensor("op_1889")]; - tensor var_1890 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1890")]; - tensor filled_window = add(x = var_1889, y = var_1890)[name = tensor("filled_window")]; - tensor var_1892 = greater_equal(x = clip_29, y = var_16)[name = tensor("op_1892")]; - tensor var_1893_dtype_0 = const()[name = tensor("op_1893_dtype_0"), val = tensor("fp32")]; - tensor var_1894 = const()[name = tensor("op_1894"), val = tensor([1, 1, 1])]; - tensor var_1893 = cast(dtype = var_1893_dtype_0, x = var_1892)[name = tensor("cast_144")]; - tensor use_roll = reshape(shape = var_1894, x = var_1893)[name = tensor("use_roll")]; - tensor var_1896 = sub(x = var_17, y = use_roll)[name = tensor("op_1896")]; - tensor var_1897 = mul(x = filled_window, y = var_1896)[name = tensor("op_1897")]; - tensor var_1898 = mul(x = rolled, y = use_roll)[name = tensor("op_1898")]; - tensor input_261 = add(x = var_1897, y = var_1898)[name = tensor("input_261")]; - tensor var_1900 = const()[name = tensor("op_1900"), val = tensor(0x1p+0)]; - tensor var_1901 = add(x = clip_29, y = var_1900)[name = tensor("op_1901")]; - tensor clip_31 = clip(alpha = var_21, beta = var_16, x = var_1901)[name = tensor("clip_31")]; - tensor x_47_axes_0 = const()[name = tensor("x_47_axes_0"), val = tensor([-1])]; - tensor x_47 = layer_norm(axes = x_47_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_261)[name = tensor("x_47")]; - tensor input_263_perm_0 = const()[name = tensor("input_263_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_105_pad_type_0 = const()[name = tensor("inputs_105_pad_type_0"), val = tensor("valid")]; - tensor inputs_105_strides_0 = const()[name = tensor("inputs_105_strides_0"), val = tensor([1])]; - tensor inputs_105_pad_0 = const()[name = tensor("inputs_105_pad_0"), val = tensor([0, 0])]; - tensor inputs_105_dilations_0 = const()[name = tensor("inputs_105_dilations_0"), val = tensor([1])]; - tensor inputs_105_groups_0 = const()[name = tensor("inputs_105_groups_0"), val = tensor(1)]; - tensor input_263 = transpose(perm = input_263_perm_0, x = x_47)[name = tensor("transpose_32")]; - tensor inputs_105 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_105_dilations_0, groups = inputs_105_groups_0, pad = inputs_105_pad_0, pad_type = inputs_105_pad_type_0, strides = inputs_105_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_263)[name = tensor("inputs_105")]; - tensor var_1922_split_sizes_0 = const()[name = tensor("op_1922_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1922_axis_0 = const()[name = tensor("op_1922_axis_0"), val = tensor(1)]; - tensor var_1922_0, tensor var_1922_1 = split(axis = var_1922_axis_0, split_sizes = var_1922_split_sizes_0, x = inputs_105)[name = tensor("op_1922")]; - tensor var_1924 = sigmoid(x = var_1922_1)[name = tensor("op_1924")]; - tensor inputs_107 = mul(x = var_1922_0, y = var_1924)[name = tensor("inputs_107")]; + tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1229)[name = tensor("x_expanded")]; + tensor var_1243 = sub(x = var_17, y = fill_mask)[name = tensor("op_1243")]; + tensor var_1244 = mul(x = window_37, y = var_1243)[name = tensor("op_1244")]; + tensor var_1245 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1245")]; + tensor filled_window = add(x = var_1244, y = var_1245)[name = tensor("filled_window")]; + tensor var_1247 = greater_equal(x = clip_29, y = var_16)[name = tensor("op_1247")]; + tensor var_1248_dtype_0 = const()[name = tensor("op_1248_dtype_0"), val = tensor("fp32")]; + tensor var_1249 = const()[name = tensor("op_1249"), val = tensor([1, 1, 1])]; + tensor var_1248 = cast(dtype = var_1248_dtype_0, x = var_1247)[name = tensor("cast_144")]; + tensor use_roll = reshape(shape = var_1249, x = var_1248)[name = tensor("use_roll")]; + tensor var_1251 = sub(x = var_17, y = use_roll)[name = tensor("op_1251")]; + tensor var_1252 = mul(x = filled_window, y = var_1251)[name = tensor("op_1252")]; + tensor var_1253 = mul(x = rolled, y = use_roll)[name = tensor("op_1253")]; + tensor window = add(x = var_1252, y = var_1253)[name = tensor("window")]; + tensor var_1255 = const()[name = tensor("op_1255"), val = tensor(0x1p+0)]; + tensor var_1256 = add(x = clip_29, y = var_1255)[name = tensor("op_1256")]; + tensor clip_31 = clip(alpha = var_21, beta = var_16, x = var_1256)[name = tensor("clip_31")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_30, interleave = input_141_interleave_0, values = (window_33, window_35, window_37, window))[name = tensor("input_141")]; + tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; + tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; + tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_33_pad_type_0 = const()[name = tensor("inputs_33_pad_type_0"), val = tensor("valid")]; + tensor inputs_33_strides_0 = const()[name = tensor("inputs_33_strides_0"), val = tensor([1])]; + tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; + tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; + tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; + tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; + tensor var_1279_split_sizes_0 = const()[name = tensor("op_1279_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1279_axis_0 = const()[name = tensor("op_1279_axis_0"), val = tensor(1)]; + tensor var_1279_0, tensor var_1279_1 = split(axis = var_1279_axis_0, split_sizes = var_1279_split_sizes_0, x = inputs_33)[name = tensor("op_1279")]; + tensor var_1281 = sigmoid(x = var_1279_1)[name = tensor("op_1281")]; + tensor inputs_35 = mul(x = var_1279_0, y = var_1281)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; tensor outputs_aug_strides_0 = const()[name = tensor("outputs_aug_strides_0"), val = tensor([1])]; tensor outputs_aug_dilations_0 = const()[name = tensor("outputs_aug_dilations_0"), val = tensor([1])]; - tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_107)[name = tensor("outputs_aug")]; - tensor input_265_begin_0 = const()[name = tensor("input_265_begin_0"), val = tensor([0, 0, 0])]; - tensor input_265_end_0 = const()[name = tensor("input_265_end_0"), val = tensor([1, 256, 16])]; - tensor input_265_end_mask_0 = const()[name = tensor("input_265_end_mask_0"), val = tensor([true, true, false])]; - tensor input_265 = slice_by_index(begin = input_265_begin_0, end = input_265_end_0, end_mask = input_265_end_mask_0, x = outputs_aug)[name = tensor("input_265")]; - tensor inputs_109 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_265)[name = tensor("inputs_109")]; - tensor input_267 = silu(x = inputs_109)[name = tensor("input_267")]; - tensor input_269_pad_type_0 = const()[name = tensor("input_269_pad_type_0"), val = tensor("valid")]; - tensor input_269_strides_0 = const()[name = tensor("input_269_strides_0"), val = tensor([1])]; - tensor input_269_pad_0 = const()[name = tensor("input_269_pad_0"), val = tensor([0, 0])]; - tensor input_269_dilations_0 = const()[name = tensor("input_269_dilations_0"), val = tensor([1])]; - tensor input_269_groups_0 = const()[name = tensor("input_269_groups_0"), val = tensor(1)]; - tensor input_269 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_269_dilations_0, groups = input_269_groups_0, pad = input_269_pad_0, pad_type = input_269_pad_type_0, strides = input_269_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_267)[name = tensor("input_269")]; - tensor conv_out_31_perm_0 = const()[name = tensor("conv_out_31_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1955_begin_0 = const()[name = tensor("op_1955_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1955_end_0 = const()[name = tensor("op_1955_end_0"), val = tensor([1, 16, 256])]; - tensor var_1955_end_mask_0 = const()[name = tensor("op_1955_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_31 = transpose(perm = conv_out_31_perm_0, x = input_269)[name = tensor("transpose_31")]; - tensor var_1955 = slice_by_index(begin = var_1955_begin_0, end = var_1955_end_0, end_mask = var_1955_end_mask_0, x = conv_out_31)[name = tensor("op_1955")]; - tensor var_1958_interleave_0 = const()[name = tensor("op_1958_interleave_0"), val = tensor(false)]; - tensor var_1958 = concat(axis = var_33, interleave = var_1958_interleave_0, values = (var_1703, var_1787, var_1871, var_1955))[name = tensor("op_1958")]; - tensor input_271 = add(x = x_39, y = var_1958)[name = tensor("input_271")]; - tensor input_273_axes_0 = const()[name = tensor("input_273_axes_0"), val = tensor([-1])]; - tensor input_273 = layer_norm(axes = input_273_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_271)[name = tensor("input_273")]; - tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_273)[name = tensor("linear_35")]; - tensor input_275 = silu(x = inputs)[name = tensor("input_275")]; - tensor input_279 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_275)[name = tensor("linear_36")]; - tensor var_1981 = const()[name = tensor("op_1981"), val = tensor(0x1p-1)]; - tensor var_1982 = mul(x = input_279, y = var_1981)[name = tensor("op_1982")]; - tensor input_281 = add(x = var_1982, y = input_271)[name = tensor("input_281")]; - tensor x_49_axes_0 = const()[name = tensor("x_49_axes_0"), val = tensor([-1])]; - tensor x_49 = layer_norm(axes = x_49_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_281)[name = tensor("x_49")]; + tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_35)[name = tensor("outputs_aug")]; + tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; + tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([4, 256, 16])]; + tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; + tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug)[name = tensor("input_145")]; + tensor inputs_37 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_35, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_145)[name = tensor("inputs_37")]; + tensor input_147 = silu(x = inputs_37)[name = tensor("input_147")]; + tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; + tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; + tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; + tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; + tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; + tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; + tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1312_begin_0 = const()[name = tensor("op_1312_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1312_end_0 = const()[name = tensor("op_1312_end_0"), val = tensor([4, 16, 256])]; + tensor var_1312_end_mask_0 = const()[name = tensor("op_1312_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_1312 = slice_by_index(begin = var_1312_begin_0, end = var_1312_end_0, end_mask = var_1312_end_mask_0, x = conv_out_7)[name = tensor("op_1312")]; + tensor var_1314_perm_0 = const()[name = tensor("op_1314_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1314 = transpose(perm = var_1314_perm_0, x = var_1312)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_1314)[name = tensor("input_151")]; + tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; + tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_35, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; + tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; + tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; + tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; + tensor var_1337 = const()[name = tensor("op_1337"), val = tensor(0x1p-1)]; + tensor var_1338 = mul(x = input_159, y = var_1337)[name = tensor("op_1338")]; + tensor input_161 = add(x = var_1338, y = input_151)[name = tensor("input_161")]; + tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; + tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_35, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; tensor cat_interleave_0 = const()[name = tensor("cat_interleave_0"), val = tensor(false)]; - tensor x_ct = transpose(perm = x_ct_perm_0, x = x_49)[name = tensor("transpose_30")]; + tensor x_ct = transpose(perm = x_ct_perm_0, x = x_25)[name = tensor("transpose_30")]; tensor cat = concat(axis = var_27, interleave = cat_interleave_0, values = (cnn_window, x_ct))[name = tensor("cat")]; tensor conv_out_pad_type_0 = const()[name = tensor("conv_out_pad_type_0"), val = tensor("valid")]; tensor conv_out_strides_0 = const()[name = tensor("conv_out_strides_0"), val = tensor([1])]; @@ -1805,44 +1346,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_2000_begin_0 = const()[name = tensor("op_2000_begin_0"), val = tensor([0, 0, 4])]; - tensor var_2000_end_0 = const()[name = tensor("op_2000_end_0"), val = tensor([1, 256, 22])]; - tensor var_2000_end_mask_0 = const()[name = tensor("op_2000_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_2000_begin_0, end = var_2000_end_0, end_mask = var_2000_end_mask_0, x = cat)[name = tensor("op_2000")]; - tensor input_283_perm_0 = const()[name = tensor("input_283_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2002 = const()[name = tensor("op_2002"), val = tensor([-1])]; - tensor input_283 = transpose(perm = input_283_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_2003 = reduce_l2_norm(axes = var_2002, keep_dims = var_36, x = input_283)[name = tensor("op_2003")]; - tensor const_24 = const()[name = tensor("const_24"), val = tensor(0x1.fffffep+127)]; - tensor clip_32 = clip(alpha = var_12, beta = const_24, x = var_2003)[name = tensor("clip_32")]; - tensor emb = real_div(x = input_283, y = clip_32)[name = tensor("emb")]; - tensor var_2007_axis_0 = const()[name = tensor("op_2007_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_2007_axis_0, values = (var_214, var_678, var_1142, nkv_1))[name = tensor("op_2007")]; - tensor var_2009_axis_0 = const()[name = tensor("op_2009_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_2009_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_2009")]; - tensor var_2011_axis_0 = const()[name = tensor("op_2011_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_2011_axis_0, values = (input_51, input_121, input_191, input_261))[name = tensor("op_2011")]; - tensor var_2013_axis_0 = const()[name = tensor("op_2013_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_2013_axis_0, values = (clip_7, clip_15, clip_23, clip_31))[name = tensor("op_2013")]; - tensor var_2023 = const()[name = tensor("op_2023"), val = tensor(0x1.5798eep-27)]; - tensor var_2029 = const()[name = tensor("op_2029"), val = tensor(0x1.4f8b58p-17)]; - tensor var_2031 = const()[name = tensor("op_2031"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_2032 = const()[name = tensor("op_2032"), val = tensor(true)]; - tensor var_2034 = const()[name = tensor("op_2034"), val = tensor(0x1p+0)]; - tensor var_2038 = const()[name = tensor("op_2038"), val = tensor(-1)]; - tensor var_2043 = const()[name = tensor("op_2043"), val = tensor(0)]; + tensor var_1356_begin_0 = const()[name = tensor("op_1356_begin_0"), val = tensor([0, 0, 4])]; + tensor var_1356_end_0 = const()[name = tensor("op_1356_end_0"), val = tensor([1, 256, 22])]; + tensor var_1356_end_mask_0 = const()[name = tensor("op_1356_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_1356_begin_0, end = var_1356_end_0, end_mask = var_1356_end_mask_0, x = cat)[name = tensor("op_1356")]; + tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1358 = const()[name = tensor("op_1358"), val = tensor([-1])]; + tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; + tensor var_1359 = reduce_l2_norm(axes = var_1358, keep_dims = var_36, x = input_163)[name = tensor("op_1359")]; + tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; + tensor clip_32 = clip(alpha = var_12, beta = const_12, x = var_1359)[name = tensor("clip_32")]; + tensor emb = real_div(x = input_163, y = clip_32)[name = tensor("emb")]; + tensor var_1363_axis_0 = const()[name = tensor("op_1363_axis_0"), val = tensor(0)]; + tensor enc_kv_new = stack(axis = var_1363_axis_0, values = (var_214, var_517, var_820, nkv_1))[name = tensor("op_1363")]; + tensor var_1365_axis_0 = const()[name = tensor("op_1365_axis_0"), val = tensor(0)]; + tensor enc_scale_new = stack(axis = var_1365_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1365")]; + tensor var_1367_axis_0 = const()[name = tensor("op_1367_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1367_axis_0, values = (window_9, window_19, window_29, window))[name = tensor("op_1367")]; + tensor var_1369_axis_0 = const()[name = tensor("op_1369_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1369_axis_0, values = (clip_7, clip_15, clip_23, clip_31))[name = tensor("op_1369")]; + tensor var_1379 = const()[name = tensor("op_1379"), val = tensor(0x1.5798eep-27)]; + tensor var_1384 = const()[name = tensor("op_1384"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1386 = const()[name = tensor("op_1386"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1387 = const()[name = tensor("op_1387"), val = tensor(true)]; + tensor var_1389 = const()[name = tensor("op_1389"), val = tensor(0x1p+0)]; + tensor var_1393 = const()[name = tensor("op_1393"), val = tensor(-1)]; + tensor var_1399 = const()[name = tensor("op_1399"), val = tensor(0)]; tensor pos = const()[name = tensor("pos"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395712)))]; - tensor var_2105_axes_0 = const()[name = tensor("op_2105_axes_0"), val = tensor([2])]; - tensor var_2105 = expand_dims(axes = var_2105_axes_0, x = emb)[name = tensor("op_2105")]; - tensor var_2106 = const()[name = tensor("op_2106"), val = tensor([1, 1, 12, 1])]; - tensor emb_exp = tile(reps = var_2106, x = var_2105)[name = tensor("emb_exp")]; - tensor input_285_interleave_0 = const()[name = tensor("input_285_interleave_0"), val = tensor(false)]; - tensor input_285 = concat(axis = var_2038, interleave = input_285_interleave_0, values = (emb_exp, pos))[name = tensor("input_285")]; - tensor x_51 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_285)[name = tensor("linear_37")]; - tensor var_2113_perm_0 = const()[name = tensor("op_2113_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2117 = const()[name = tensor("op_2117"), val = tensor([12, 4, 256])]; - tensor var_2113 = transpose(perm = var_2113_perm_0, x = x_51)[name = tensor("transpose_28")]; - tensor x_53 = reshape(shape = var_2117, x = var_2113)[name = tensor("x_53")]; + tensor var_1461_axes_0 = const()[name = tensor("op_1461_axes_0"), val = tensor([2])]; + tensor var_1461 = expand_dims(axes = var_1461_axes_0, x = emb)[name = tensor("op_1461")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 12, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1461)[name = tensor("emb_exp")]; + tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; + tensor input_165 = concat(axis = var_1393, interleave = input_165_interleave_0, values = (emb_exp, pos))[name = tensor("input_165")]; + tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; + tensor var_1469_perm_0 = const()[name = tensor("op_1469_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1473 = const()[name = tensor("op_1473"), val = tensor([12, 4, 256])]; + tensor var_1469 = transpose(perm = var_1469_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1473, x = var_1469)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 12, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1853,132 +1394,132 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_2125 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_53)[name = tensor("linear_38")]; - tensor var_2126 = const()[name = tensor("op_2126"), val = tensor([12, 4, 4, 64])]; - tensor var_2127 = reshape(shape = var_2126, x = var_2125)[name = tensor("op_2127")]; + tensor var_1481 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1482 = const()[name = tensor("op_1482"), val = tensor([12, 4, 4, 64])]; + tensor var_1483 = reshape(shape = var_1482, x = var_1481)[name = tensor("op_1483")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2131 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_53)[name = tensor("linear_39")]; - tensor var_2132 = const()[name = tensor("op_2132"), val = tensor(0x1p-3)]; - tensor var_2133 = mul(x = var_2131, y = var_2132)[name = tensor("op_2133")]; - tensor var_2134 = const()[name = tensor("op_2134"), val = tensor([12, 4, 4, 64])]; - tensor var_2135 = reshape(shape = var_2134, x = var_2133)[name = tensor("op_2135")]; + tensor var_1487 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1488 = const()[name = tensor("op_1488"), val = tensor(0x1p-3)]; + tensor var_1489 = mul(x = var_1487, y = var_1488)[name = tensor("op_1489")]; + tensor var_1490 = const()[name = tensor("op_1490"), val = tensor([12, 4, 4, 64])]; + tensor var_1491 = reshape(shape = var_1490, x = var_1489)[name = tensor("op_1491")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2139 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_53)[name = tensor("linear_40")]; - tensor var_2140 = const()[name = tensor("op_2140"), val = tensor([12, 4, 4, 64])]; - tensor var_2141 = reshape(shape = var_2140, x = var_2139)[name = tensor("op_2141")]; + tensor var_1495 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1496 = const()[name = tensor("op_1496"), val = tensor([12, 4, 4, 64])]; + tensor var_1497 = reshape(shape = var_1496, x = var_1495)[name = tensor("op_1497")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_289 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_53)[name = tensor("linear_41")]; + tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_2043, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1399, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; - tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; - tensor clip_33 = clip(alpha = var_2034, beta = const_32, x = s_t_9)[name = tensor("clip_33")]; + tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; + tensor clip_33 = clip(alpha = var_1389, beta = const_20, x = s_t_9)[name = tensor("clip_33")]; tensor sqrt_s_t_9 = sqrt(x = clip_33)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_2135)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_2127)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1491)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1483)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_2153 = const()[name = tensor("op_2153"), val = tensor([1, 4])]; - tensor var_2154 = reshape(shape = var_2153, x = valid_mask)[name = tensor("op_2154")]; - tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_2154)[name = tensor("causal_with_valid_1")]; - tensor var_2156 = const()[name = tensor("op_2156"), val = tensor([4, 1])]; - tensor var_2157 = reshape(shape = var_2156, x = sqrt_s_t_9)[name = tensor("op_2157")]; - tensor M_9 = real_div(x = causal_with_valid_1, y = var_2157)[name = tensor("M_9")]; - tensor var_2159 = mul(x = qk_9, y = M_9)[name = tensor("op_2159")]; + tensor var_1509 = const()[name = tensor("op_1509"), val = tensor([1, 4])]; + tensor var_1510 = reshape(shape = var_1509, x = valid_mask)[name = tensor("op_1510")]; + tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1510)[name = tensor("causal_with_valid_1")]; + tensor var_1512 = const()[name = tensor("op_1512"), val = tensor([4, 1])]; + tensor var_1513 = reshape(shape = var_1512, x = sqrt_s_t_9)[name = tensor("op_1513")]; + tensor M_9 = real_div(x = causal_with_valid_1, y = var_1513)[name = tensor("M_9")]; + tensor var_1515 = mul(x = qk_9, y = M_9)[name = tensor("op_1515")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_2141)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_2159, y = v_9)[name = tensor("inner_9")]; - tensor var_2161_transpose_x_0 = const()[name = tensor("op_2161_transpose_x_0"), val = tensor(false)]; - tensor var_2161_transpose_y_0 = const()[name = tensor("op_2161_transpose_y_0"), val = tensor(false)]; - tensor var_2161 = matmul(transpose_x = var_2161_transpose_x_0, transpose_y = var_2161_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_2161")]; - tensor var_2162 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_2162")]; - tensor var_2163 = const()[name = tensor("op_2163"), val = tensor([1, 1, 4, 1])]; - tensor var_2164 = reshape(shape = var_2163, x = var_2162)[name = tensor("op_2164")]; - tensor cross_9 = mul(x = var_2161, y = var_2164)[name = tensor("cross_9")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1497)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1515, y = v_9)[name = tensor("inner_9")]; + tensor var_1517_transpose_x_0 = const()[name = tensor("op_1517_transpose_x_0"), val = tensor(false)]; + tensor var_1517_transpose_y_0 = const()[name = tensor("op_1517_transpose_y_0"), val = tensor(false)]; + tensor var_1517 = matmul(transpose_x = var_1517_transpose_x_0, transpose_y = var_1517_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1517")]; + tensor var_1518 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1518")]; + tensor var_1519 = const()[name = tensor("op_1519"), val = tensor([1, 1, 4, 1])]; + tensor var_1520 = reshape(shape = var_1519, x = var_1518)[name = tensor("op_1520")]; + tensor cross_9 = mul(x = var_1517, y = var_1520)[name = tensor("cross_9")]; tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; - tensor var_2167 = const()[name = tensor("op_2167"), val = tensor([1, 1, 4, 1])]; - tensor var_2168 = reshape(shape = var_2167, x = valid_mask)[name = tensor("op_2168")]; - tensor v_masked_1 = mul(x = v_9, y = var_2168)[name = tensor("v_masked_1")]; - tensor var_2170 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_2170")]; - tensor var_2172_transpose_x_1 = const()[name = tensor("op_2172_transpose_x_1"), val = tensor(true)]; - tensor var_2172_transpose_y_1 = const()[name = tensor("op_2172_transpose_y_1"), val = tensor(false)]; - tensor var_2172 = matmul(transpose_x = var_2172_transpose_x_1, transpose_y = var_2172_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_2172")]; - tensor new_kv_unnorm_9 = add(x = var_2170, y = var_2172)[name = tensor("new_kv_unnorm_9")]; - tensor var_2174_keep_dims_0 = const()[name = tensor("op_2174_keep_dims_0"), val = tensor(false)]; - tensor var_2174 = reduce_sum(keep_dims = var_2174_keep_dims_0, x = valid_mask)[name = tensor("op_2174")]; - tensor var_2175 = const()[name = tensor("op_2175"), val = tensor([1])]; - tensor var_2176 = reshape(shape = var_2175, x = var_2174)[name = tensor("op_2176")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_2176)[name = tensor("new_scale_9")]; - tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; - tensor clip_34 = clip(alpha = var_2034, beta = const_33, x = new_scale_9)[name = tensor("clip_34")]; + tensor var_1523 = const()[name = tensor("op_1523"), val = tensor([1, 1, 4, 1])]; + tensor var_1524 = reshape(shape = var_1523, x = valid_mask)[name = tensor("op_1524")]; + tensor v_masked_1 = mul(x = v_9, y = var_1524)[name = tensor("v_masked_1")]; + tensor var_1526 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1526")]; + tensor var_1528_transpose_x_1 = const()[name = tensor("op_1528_transpose_x_1"), val = tensor(true)]; + tensor var_1528_transpose_y_1 = const()[name = tensor("op_1528_transpose_y_1"), val = tensor(false)]; + tensor var_1528 = matmul(transpose_x = var_1528_transpose_x_1, transpose_y = var_1528_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1528")]; + tensor new_kv_unnorm_9 = add(x = var_1526, y = var_1528)[name = tensor("new_kv_unnorm_9")]; + tensor var_1530_keep_dims_0 = const()[name = tensor("op_1530_keep_dims_0"), val = tensor(false)]; + tensor var_1530 = reduce_sum(keep_dims = var_1530_keep_dims_0, x = valid_mask)[name = tensor("op_1530")]; + tensor var_1531 = const()[name = tensor("op_1531"), val = tensor([1])]; + tensor var_1532 = reshape(shape = var_1531, x = var_1530)[name = tensor("op_1532")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1532)[name = tensor("new_scale_9")]; + tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; + tensor clip_34 = clip(alpha = var_1389, beta = const_21, x = new_scale_9)[name = tensor("clip_34")]; tensor sqrt_new_scale_1 = sqrt(x = clip_34)[name = tensor("sqrt_new_scale_1")]; - tensor var_2180 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_2180")]; - tensor var_2181_perm_0 = const()[name = tensor("op_2181_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1536 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1536")]; + tensor var_1537_perm_0 = const()[name = tensor("op_1537_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_2181 = transpose(perm = var_2181_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_2031, x = var_2181)[name = tensor("out_27")]; - tensor var_2185 = const()[name = tensor("op_2185"), val = tensor([12, 4, 256])]; - tensor out_29 = reshape(shape = var_2185, x = out_27)[name = tensor("out_29")]; - tensor var_2187 = silu(x = input_289)[name = tensor("op_2187")]; - tensor input_291 = mul(x = var_2187, y = out_29)[name = tensor("input_291")]; - tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_291)[name = tensor("linear_42")]; - tensor input_293 = add(x = x_53, y = ret_out_9)[name = tensor("input_293")]; + tensor var_1537 = transpose(perm = var_1537_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1386, x = var_1537)[name = tensor("out_27")]; + tensor var_1541 = const()[name = tensor("op_1541"), val = tensor([12, 4, 256])]; + tensor out_29 = reshape(shape = var_1541, x = out_27)[name = tensor("out_29")]; + tensor var_1543 = silu(x = input_169)[name = tensor("op_1543")]; + tensor input_171 = mul(x = var_1543, y = out_29)[name = tensor("input_171")]; + tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; + tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_2029, gamma = decoder_norm11_0_weight, x = input_293)[name = tensor("xt_1")]; - tensor var_2197 = const()[name = tensor("op_2197"), val = tensor([1, 12, 4, 256])]; - tensor var_2198 = reshape(shape = var_2197, x = xt_1)[name = tensor("op_2198")]; - tensor var_2199_perm_0 = const()[name = tensor("op_2199_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2202 = const()[name = tensor("op_2202"), val = tensor([4, 12, 256])]; - tensor var_2199 = transpose(perm = var_2199_perm_0, x = var_2198)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_2202, x = var_2199)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1384, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1553 = const()[name = tensor("op_1553"), val = tensor([1, 12, 4, 256])]; + tensor var_1554 = reshape(shape = var_1553, x = xt_1)[name = tensor("op_1554")]; + tensor var_1555_perm_0 = const()[name = tensor("op_1555_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1558 = const()[name = tensor("op_1558"), val = tensor([4, 12, 256])]; + tensor var_1555 = transpose(perm = var_1555_perm_0, x = var_1554)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1558, x = var_1555)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_2225 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_16 = const()[name = tensor("concat_16"), val = tensor([12, 4, 3, 256])]; - tensor var_2227 = reshape(shape = concat_16, x = var_2225)[name = tensor("op_2227")]; - tensor var_2228_axes_0 = const()[name = tensor("op_2228_axes_0"), val = tensor([0])]; - tensor var_2228 = expand_dims(axes = var_2228_axes_0, x = var_2227)[name = tensor("op_2228")]; - tensor var_2229_perm_0 = const()[name = tensor("op_2229_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2230_axes_0 = const()[name = tensor("op_2230_axes_0"), val = tensor([-2])]; - tensor var_2229 = transpose(perm = var_2229_perm_0, x = var_2228)[name = tensor("transpose_21")]; - tensor var_2230 = squeeze(axes = var_2230_axes_0, x = var_2229)[name = tensor("op_2230")]; + tensor var_1581 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_17 = const()[name = tensor("concat_17"), val = tensor([12, 4, 3, 256])]; + tensor var_1583 = reshape(shape = concat_17, x = var_1581)[name = tensor("op_1583")]; + tensor var_1584_axes_0 = const()[name = tensor("op_1584_axes_0"), val = tensor([0])]; + tensor var_1584 = expand_dims(axes = var_1584_axes_0, x = var_1583)[name = tensor("op_1584")]; + tensor var_1585_perm_0 = const()[name = tensor("op_1585_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1586_axes_0 = const()[name = tensor("op_1586_axes_0"), val = tensor([-2])]; + tensor var_1585 = transpose(perm = var_1585_perm_0, x = var_1584)[name = tensor("transpose_21")]; + tensor var_1586 = squeeze(axes = var_1586_axes_0, x = var_1585)[name = tensor("op_1586")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 12, 4, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_2230)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1586)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 12, 4, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_2230)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1586)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 12, 4, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_2230)[name = tensor("v_11")]; - tensor var_2238 = const()[name = tensor("op_2238"), val = tensor([12, 16, 64])]; - tensor var_2239 = reshape(shape = var_2238, x = q_11)[name = tensor("op_2239")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1586)[name = tensor("v_11")]; + tensor var_1594 = const()[name = tensor("op_1594"), val = tensor([12, 16, 64])]; + tensor var_1595 = reshape(shape = var_1594, x = q_11)[name = tensor("op_1595")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2245 = const()[name = tensor("op_2245"), val = tensor([12, 16, 64])]; - tensor var_2246 = reshape(shape = var_2245, x = k_11)[name = tensor("op_2246")]; + tensor var_1601 = const()[name = tensor("op_1601"), val = tensor([12, 16, 64])]; + tensor var_1602 = reshape(shape = var_1601, x = k_11)[name = tensor("op_1602")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2252 = const()[name = tensor("op_2252"), val = tensor([12, 16, 64])]; - tensor var_2253 = reshape(shape = var_2252, x = v_11)[name = tensor("op_2253")]; + tensor var_1608 = const()[name = tensor("op_1608"), val = tensor([12, 16, 64])]; + tensor var_1609 = reshape(shape = var_1608, x = v_11)[name = tensor("op_1609")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2256 = const()[name = tensor("op_2256"), val = tensor([4, 4, 12, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_2239)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_2256, x = q_13)[name = tensor("q_15")]; - tensor var_2258 = const()[name = tensor("op_2258"), val = tensor([4, 4, 12, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_2246)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_2258, x = k_13)[name = tensor("k_15")]; - tensor var_2260 = const()[name = tensor("op_2260"), val = tensor([4, 4, 12, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_2253)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_2260, x = v_13)[name = tensor("v_15")]; + tensor var_1612 = const()[name = tensor("op_1612"), val = tensor([4, 4, 12, 64])]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1595)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1612, x = q_13)[name = tensor("q_15")]; + tensor var_1614 = const()[name = tensor("op_1614"), val = tensor([4, 4, 12, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1602)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1614, x = k_13)[name = tensor("k_15")]; + tensor var_1616 = const()[name = tensor("op_1616"), val = tensor([4, 4, 12, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1609)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1616, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -1989,30 +1530,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_2263 = const()[name = tensor("op_2263"), val = tensor([2, 0, 1, 3])]; - tensor var_2268 = const()[name = tensor("op_2268"), val = tensor([48, 256])]; - tensor var_2264 = transpose(perm = var_2263, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_2268, x = var_2264)[name = tensor("attn_output_3")]; + tensor var_1619 = const()[name = tensor("op_1619"), val = tensor([2, 0, 1, 3])]; + tensor var_1624 = const()[name = tensor("op_1624"), val = tensor([48, 256])]; + tensor var_1620 = transpose(perm = var_1619, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1624, x = var_1620)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_2272 = const()[name = tensor("op_2272"), val = tensor([12, 4, 256])]; - tensor attn_output_7 = reshape(shape = var_2272, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1628 = const()[name = tensor("op_1628"), val = tensor([12, 4, 256])]; + tensor attn_output_7 = reshape(shape = var_1628, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; - tensor input_295 = add(x = query_1, y = sa2_out_1)[name = tensor("input_295")]; - tensor input_297_axes_0 = const()[name = tensor("input_297_axes_0"), val = tensor([-1])]; - tensor input_297 = layer_norm(axes = input_297_axes_0, beta = decoder_norm21_0_bias, epsilon = var_2029, gamma = decoder_norm21_0_weight, x = input_295)[name = tensor("input_297")]; - tensor input_299 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_297)[name = tensor("linear_45")]; - tensor input_301 = relu(x = input_299)[name = tensor("input_301")]; - tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_301)[name = tensor("linear_46")]; - tensor input_303 = add(x = input_297, y = ffn_out_1)[name = tensor("input_303")]; + tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; + tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1384, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; + tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; + tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; + tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_2029, gamma = decoder_norm22_0_weight, x = input_303)[name = tensor("xt_3")]; - tensor var_2292 = const()[name = tensor("op_2292"), val = tensor([1, 4, 12, 256])]; - tensor x_55 = reshape(shape = var_2292, x = xt_3)[name = tensor("x_55")]; - tensor var_2294_perm_0 = const()[name = tensor("op_2294_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2298 = const()[name = tensor("op_2298"), val = tensor([12, 4, 256])]; - tensor var_2294 = transpose(perm = var_2294_perm_0, x = x_55)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_2298, x = var_2294)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1384, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1648 = const()[name = tensor("op_1648"), val = tensor([1, 4, 12, 256])]; + tensor x_31 = reshape(shape = var_1648, x = xt_3)[name = tensor("x_31")]; + tensor var_1650_perm_0 = const()[name = tensor("op_1650_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1654 = const()[name = tensor("op_1654"), val = tensor([12, 4, 256])]; + tensor var_1650 = transpose(perm = var_1650_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1654, x = var_1650)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 12, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -2023,120 +1564,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_2306 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_2307 = const()[name = tensor("op_2307"), val = tensor([12, 4, 4, 64])]; - tensor var_2308 = reshape(shape = var_2307, x = var_2306)[name = tensor("op_2308")]; + tensor var_1662 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1663 = const()[name = tensor("op_1663"), val = tensor([12, 4, 4, 64])]; + tensor var_1664 = reshape(shape = var_1663, x = var_1662)[name = tensor("op_1664")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2312 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_2313 = const()[name = tensor("op_2313"), val = tensor(0x1p-3)]; - tensor var_2314 = mul(x = var_2312, y = var_2313)[name = tensor("op_2314")]; - tensor var_2315 = const()[name = tensor("op_2315"), val = tensor([12, 4, 4, 64])]; - tensor var_2316 = reshape(shape = var_2315, x = var_2314)[name = tensor("op_2316")]; + tensor var_1668 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1669 = const()[name = tensor("op_1669"), val = tensor(0x1p-3)]; + tensor var_1670 = mul(x = var_1668, y = var_1669)[name = tensor("op_1670")]; + tensor var_1671 = const()[name = tensor("op_1671"), val = tensor([12, 4, 4, 64])]; + tensor var_1672 = reshape(shape = var_1671, x = var_1670)[name = tensor("op_1672")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2320 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_2321 = const()[name = tensor("op_2321"), val = tensor([12, 4, 4, 64])]; - tensor var_2322 = reshape(shape = var_2321, x = var_2320)[name = tensor("op_2322")]; + tensor var_1676 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1677 = const()[name = tensor("op_1677"), val = tensor([12, 4, 4, 64])]; + tensor var_1678 = reshape(shape = var_1677, x = var_1676)[name = tensor("op_1678")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_307 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; + tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; - tensor const_44 = const()[name = tensor("const_44"), val = tensor(0x1.fffffep+127)]; - tensor clip_35 = clip(alpha = var_2034, beta = const_44, x = s_t)[name = tensor("clip_35")]; + tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; + tensor clip_35 = clip(alpha = var_1389, beta = const_32, x = s_t)[name = tensor("clip_35")]; tensor sqrt_s_t = sqrt(x = clip_35)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_2316)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_2308)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1672)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1664)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_2337 = const()[name = tensor("op_2337"), val = tensor([4, 1])]; - tensor var_2338 = reshape(shape = var_2337, x = sqrt_s_t)[name = tensor("op_2338")]; - tensor M = real_div(x = causal_with_valid_1, y = var_2338)[name = tensor("M")]; - tensor var_2340 = mul(x = qk, y = M)[name = tensor("op_2340")]; + tensor var_1693 = const()[name = tensor("op_1693"), val = tensor([4, 1])]; + tensor var_1694 = reshape(shape = var_1693, x = sqrt_s_t)[name = tensor("op_1694")]; + tensor M = real_div(x = causal_with_valid_1, y = var_1694)[name = tensor("M")]; + tensor var_1696 = mul(x = qk, y = M)[name = tensor("op_1696")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_2322)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_2340, y = v_17)[name = tensor("inner")]; - tensor var_2342_transpose_x_0 = const()[name = tensor("op_2342_transpose_x_0"), val = tensor(false)]; - tensor var_2342_transpose_y_0 = const()[name = tensor("op_2342_transpose_y_0"), val = tensor(false)]; - tensor var_2342 = matmul(transpose_x = var_2342_transpose_x_0, transpose_y = var_2342_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_2342")]; - tensor var_2343 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_2343")]; - tensor var_2344 = const()[name = tensor("op_2344"), val = tensor([1, 1, 4, 1])]; - tensor var_2345 = reshape(shape = var_2344, x = var_2343)[name = tensor("op_2345")]; - tensor cross = mul(x = var_2342, y = var_2345)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1678)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1696, y = v_17)[name = tensor("inner")]; + tensor var_1698_transpose_x_0 = const()[name = tensor("op_1698_transpose_x_0"), val = tensor(false)]; + tensor var_1698_transpose_y_0 = const()[name = tensor("op_1698_transpose_y_0"), val = tensor(false)]; + tensor var_1698 = matmul(transpose_x = var_1698_transpose_x_0, transpose_y = var_1698_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1698")]; + tensor var_1699 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1699")]; + tensor var_1700 = const()[name = tensor("op_1700"), val = tensor([1, 1, 4, 1])]; + tensor var_1701 = reshape(shape = var_1700, x = var_1699)[name = tensor("op_1701")]; + tensor cross = mul(x = var_1698, y = var_1701)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_2168)[name = tensor("v_masked")]; - tensor var_2351 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_2351")]; - tensor var_2353_transpose_x_1 = const()[name = tensor("op_2353_transpose_x_1"), val = tensor(true)]; - tensor var_2353_transpose_y_1 = const()[name = tensor("op_2353_transpose_y_1"), val = tensor(false)]; - tensor var_2353 = matmul(transpose_x = var_2353_transpose_x_1, transpose_y = var_2353_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_2353")]; - tensor new_kv_unnorm = add(x = var_2351, y = var_2353)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_2176)[name = tensor("new_scale")]; - tensor const_45 = const()[name = tensor("const_45"), val = tensor(0x1.fffffep+127)]; - tensor clip_36 = clip(alpha = var_2034, beta = const_45, x = new_scale)[name = tensor("clip_36")]; + tensor v_masked = mul(x = v_17, y = var_1524)[name = tensor("v_masked")]; + tensor var_1707 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1707")]; + tensor var_1709_transpose_x_1 = const()[name = tensor("op_1709_transpose_x_1"), val = tensor(true)]; + tensor var_1709_transpose_y_1 = const()[name = tensor("op_1709_transpose_y_1"), val = tensor(false)]; + tensor var_1709 = matmul(transpose_x = var_1709_transpose_x_1, transpose_y = var_1709_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1709")]; + tensor new_kv_unnorm = add(x = var_1707, y = var_1709)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1532)[name = tensor("new_scale")]; + tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; + tensor clip_36 = clip(alpha = var_1389, beta = const_33, x = new_scale)[name = tensor("clip_36")]; tensor sqrt_new_scale = sqrt(x = clip_36)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_2362_perm_0 = const()[name = tensor("op_2362_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1718_perm_0 = const()[name = tensor("op_1718_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_2362 = transpose(perm = var_2362_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_2031, x = var_2362)[name = tensor("out_33")]; - tensor var_2366 = const()[name = tensor("op_2366"), val = tensor([12, 4, 256])]; - tensor out = reshape(shape = var_2366, x = out_33)[name = tensor("out")]; - tensor var_2368 = silu(x = input_307)[name = tensor("op_2368")]; - tensor input_309 = mul(x = var_2368, y = out)[name = tensor("input_309")]; - tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_309)[name = tensor("linear_51")]; - tensor input_311 = add(x = x, y = ret_out)[name = tensor("input_311")]; + tensor var_1718 = transpose(perm = var_1718_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1386, x = var_1718)[name = tensor("out_33")]; + tensor var_1722 = const()[name = tensor("op_1722"), val = tensor([12, 4, 256])]; + tensor out = reshape(shape = var_1722, x = out_33)[name = tensor("out")]; + tensor var_1724 = silu(x = input_187)[name = tensor("op_1724")]; + tensor input_189 = mul(x = var_1724, y = out)[name = tensor("input_189")]; + tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; + tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_2029, gamma = decoder_norm11_1_weight, x = input_311)[name = tensor("xt_5")]; - tensor var_2378 = const()[name = tensor("op_2378"), val = tensor([1, 12, 4, 256])]; - tensor var_2379 = reshape(shape = var_2378, x = xt_5)[name = tensor("op_2379")]; - tensor var_2380_perm_0 = const()[name = tensor("op_2380_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2383 = const()[name = tensor("op_2383"), val = tensor([4, 12, 256])]; - tensor var_2380 = transpose(perm = var_2380_perm_0, x = var_2379)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_2383, x = var_2380)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1384, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1734 = const()[name = tensor("op_1734"), val = tensor([1, 12, 4, 256])]; + tensor var_1735 = reshape(shape = var_1734, x = xt_5)[name = tensor("op_1735")]; + tensor var_1736_perm_0 = const()[name = tensor("op_1736_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1739 = const()[name = tensor("op_1739"), val = tensor([4, 12, 256])]; + tensor var_1736 = transpose(perm = var_1736_perm_0, x = var_1735)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1739, x = var_1736)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_2406 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_17 = const()[name = tensor("concat_17"), val = tensor([12, 4, 3, 256])]; - tensor var_2408 = reshape(shape = concat_17, x = var_2406)[name = tensor("op_2408")]; - tensor var_2409_axes_0 = const()[name = tensor("op_2409_axes_0"), val = tensor([0])]; - tensor var_2409 = expand_dims(axes = var_2409_axes_0, x = var_2408)[name = tensor("op_2409")]; - tensor var_2410_perm_0 = const()[name = tensor("op_2410_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2411_axes_0 = const()[name = tensor("op_2411_axes_0"), val = tensor([-2])]; - tensor var_2410 = transpose(perm = var_2410_perm_0, x = var_2409)[name = tensor("transpose_8")]; - tensor var_2411 = squeeze(axes = var_2411_axes_0, x = var_2410)[name = tensor("op_2411")]; + tensor var_1762 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_18 = const()[name = tensor("concat_18"), val = tensor([12, 4, 3, 256])]; + tensor var_1764 = reshape(shape = concat_18, x = var_1762)[name = tensor("op_1764")]; + tensor var_1765_axes_0 = const()[name = tensor("op_1765_axes_0"), val = tensor([0])]; + tensor var_1765 = expand_dims(axes = var_1765_axes_0, x = var_1764)[name = tensor("op_1765")]; + tensor var_1766_perm_0 = const()[name = tensor("op_1766_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1767_axes_0 = const()[name = tensor("op_1767_axes_0"), val = tensor([-2])]; + tensor var_1766 = transpose(perm = var_1766_perm_0, x = var_1765)[name = tensor("transpose_8")]; + tensor var_1767 = squeeze(axes = var_1767_axes_0, x = var_1766)[name = tensor("op_1767")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 12, 4, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_2411)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1767)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 12, 4, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_2411)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1767)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 12, 4, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_2411)[name = tensor("v_19")]; - tensor var_2419 = const()[name = tensor("op_2419"), val = tensor([12, 16, 64])]; - tensor var_2420 = reshape(shape = var_2419, x = q_19)[name = tensor("op_2420")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1767)[name = tensor("v_19")]; + tensor var_1775 = const()[name = tensor("op_1775"), val = tensor([12, 16, 64])]; + tensor var_1776 = reshape(shape = var_1775, x = q_19)[name = tensor("op_1776")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2426 = const()[name = tensor("op_2426"), val = tensor([12, 16, 64])]; - tensor var_2427 = reshape(shape = var_2426, x = k_19)[name = tensor("op_2427")]; + tensor var_1782 = const()[name = tensor("op_1782"), val = tensor([12, 16, 64])]; + tensor var_1783 = reshape(shape = var_1782, x = k_19)[name = tensor("op_1783")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2433 = const()[name = tensor("op_2433"), val = tensor([12, 16, 64])]; - tensor var_2434 = reshape(shape = var_2433, x = v_19)[name = tensor("op_2434")]; + tensor var_1789 = const()[name = tensor("op_1789"), val = tensor([12, 16, 64])]; + tensor var_1790 = reshape(shape = var_1789, x = v_19)[name = tensor("op_1790")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2437 = const()[name = tensor("op_2437"), val = tensor([4, 4, 12, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_2420)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_2437, x = q_21)[name = tensor("q")]; - tensor var_2439 = const()[name = tensor("op_2439"), val = tensor([4, 4, 12, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_2427)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_2439, x = k_21)[name = tensor("k")]; - tensor var_2441 = const()[name = tensor("op_2441"), val = tensor([4, 4, 12, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_2434)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_2441, x = v_21)[name = tensor("v")]; + tensor var_1793 = const()[name = tensor("op_1793"), val = tensor([4, 4, 12, 64])]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1776)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1793, x = q_21)[name = tensor("q")]; + tensor var_1795 = const()[name = tensor("op_1795"), val = tensor([4, 4, 12, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1783)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1795, x = k_21)[name = tensor("k")]; + tensor var_1797 = const()[name = tensor("op_1797"), val = tensor([4, 4, 12, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1790)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1797, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -2147,50 +1688,50 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_2444 = const()[name = tensor("op_2444"), val = tensor([2, 0, 1, 3])]; - tensor var_2449 = const()[name = tensor("op_2449"), val = tensor([48, 256])]; - tensor var_2445 = transpose(perm = var_2444, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_2449, x = var_2445)[name = tensor("attn_output_11")]; + tensor var_1800 = const()[name = tensor("op_1800"), val = tensor([2, 0, 1, 3])]; + tensor var_1805 = const()[name = tensor("op_1805"), val = tensor([48, 256])]; + tensor var_1801 = transpose(perm = var_1800, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1805, x = var_1801)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_2453 = const()[name = tensor("op_2453"), val = tensor([12, 4, 256])]; - tensor attn_output = reshape(shape = var_2453, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1809 = const()[name = tensor("op_1809"), val = tensor([12, 4, 256])]; + tensor attn_output = reshape(shape = var_1809, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; - tensor input_313 = add(x = query_5, y = sa2_out)[name = tensor("input_313")]; - tensor input_315_axes_0 = const()[name = tensor("input_315_axes_0"), val = tensor([-1])]; - tensor input_315 = layer_norm(axes = input_315_axes_0, beta = decoder_norm21_1_bias, epsilon = var_2029, gamma = decoder_norm21_1_weight, x = input_313)[name = tensor("input_315")]; - tensor input_317 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_315)[name = tensor("linear_54")]; - tensor input_319 = relu(x = input_317)[name = tensor("input_319")]; - tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_319)[name = tensor("linear_55")]; - tensor input_321 = add(x = input_315, y = ffn_out)[name = tensor("input_321")]; + tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; + tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1384, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; + tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; + tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; + tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_2029, gamma = decoder_norm22_1_weight, x = input_321)[name = tensor("xt")]; - tensor var_2473 = const()[name = tensor("op_2473"), val = tensor([1, 4, 12, 256])]; - tensor input = reshape(shape = var_2473, x = xt)[name = tensor("input")]; - tensor var_2475 = const()[name = tensor("op_2475"), val = tensor([-1])]; - tensor var_2476 = reduce_l2_norm(axes = var_2475, keep_dims = var_2032, x = input)[name = tensor("op_2476")]; - tensor const_54 = const()[name = tensor("const_54"), val = tensor(0x1.fffffep+127)]; - tensor clip_37 = clip(alpha = var_2023, beta = const_54, x = var_2476)[name = tensor("clip_37")]; - tensor var_2478 = real_div(x = input, y = clip_37)[name = tensor("op_2478")]; - tensor concat_21 = const()[name = tensor("concat_21"), val = tensor([4, 1, 256])]; - tensor reshape_0 = reshape(shape = concat_21, x = emb)[name = tensor("reshape_0")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1384, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1829 = const()[name = tensor("op_1829"), val = tensor([1, 4, 12, 256])]; + tensor input = reshape(shape = var_1829, x = xt)[name = tensor("input")]; + tensor var_1831 = const()[name = tensor("op_1831"), val = tensor([-1])]; + tensor var_1832 = reduce_l2_norm(axes = var_1831, keep_dims = var_1387, x = input)[name = tensor("op_1832")]; + tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; + tensor clip_37 = clip(alpha = var_1379, beta = const_42, x = var_1832)[name = tensor("clip_37")]; + tensor var_1834 = real_div(x = input, y = clip_37)[name = tensor("op_1834")]; + tensor concat_22 = const()[name = tensor("concat_22"), val = tensor([4, 1, 256])]; + tensor reshape_0 = reshape(shape = concat_22, x = emb)[name = tensor("reshape_0")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_22 = const()[name = tensor("concat_22"), val = tensor([4, 256, 12])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_2478)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_22, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_23 = const()[name = tensor("concat_23"), val = tensor([4, 256, 12])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1834)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_23, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = reshape_0, y = reshape_1)[name = tensor("matmul_2")]; - tensor concat_25 = const()[name = tensor("concat_25"), val = tensor([1, 4, 12])]; - tensor reshape_2 = reshape(shape = concat_25, x = matmul_2)[name = tensor("reshape_2")]; + tensor concat_26 = const()[name = tensor("concat_26"), val = tensor([1, 4, 12])]; + tensor reshape_2 = reshape(shape = concat_26, x = matmul_2)[name = tensor("reshape_2")]; tensor output_begin_0 = const()[name = tensor("output_begin_0"), val = tensor([0, 0, 1])]; tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 4, 11])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = reshape_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_2482")]; - tensor var_2484_axis_0 = const()[name = tensor("op_2484_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_2484_axis_0, values = (var_2180, nkv))[name = tensor("op_2484")]; - tensor var_2486_axis_0 = const()[name = tensor("op_2486_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_2486_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_2486")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1838")]; + tensor var_1840_axis_0 = const()[name = tensor("op_1840_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1840_axis_0, values = (var_1536, nkv))[name = tensor("op_1840")]; + tensor var_1842_axis_0 = const()[name = tensor("op_1842_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1842_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1842")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/DIHARD III/ls_eend_dih3_400ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/DIHARD III/ls_eend_dih3_400ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index 1a4828dd84799ae61417949d7dd849645600f25f..3b0ec6590294256e0ae169a2bcdd1662cc77cf4a 100644 --- a/optimized/DIHARD III/ls_eend_dih3_400ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/DIHARD III/ls_eend_dih3_400ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:278a6cd09e5d100e369b0b7d25d2dd0177d7a0e92197ff50c150c6faf1ff0f54 -size 311061 +oid sha256:68391a10c0549fd2875304bfe1213ceb7426d62c5caff9d8c0443bc50ce00611 +size 238966 diff --git a/optimized/DIHARD III/ls_eend_dih3_400ms.mlpackage/Manifest.json b/optimized/DIHARD III/ls_eend_dih3_400ms.mlpackage/Manifest.json index ad0240166097a7cc9f4769de3f50027fa2568896..b43ec4eaacf04c4cf62b74eb4a405ff50ad08c9a 100644 --- a/optimized/DIHARD III/ls_eend_dih3_400ms.mlpackage/Manifest.json +++ b/optimized/DIHARD III/ls_eend_dih3_400ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "26BABA27-A8CE-4663-B40E-377637DC8718": { + "946AFC7F-677A-4203-9F6E-0A1807413FB2": { "author": "com.apple.CoreML", "description": "CoreML Model Specification", "name": "model.mlmodel", "path": "com.apple.CoreML/model.mlmodel" }, - "8BAEB578-9BD3-4D6F-A391-CFE72A2882EB": { + "A142DE2A-A31E-4997-A7DB-23E2D881B794": { "author": "com.apple.CoreML", "description": "CoreML Model Weights", "name": "weights", "path": "com.apple.CoreML/weights" } }, - "rootModelIdentifier": "26BABA27-A8CE-4663-B40E-377637DC8718" + "rootModelIdentifier": "946AFC7F-677A-4203-9F6E-0A1807413FB2" } diff --git a/optimized/DIHARD III/ls_eend_dih3_500ms.mlmodelc/analytics/coremldata.bin b/optimized/DIHARD III/ls_eend_dih3_500ms.mlmodelc/analytics/coremldata.bin index 403bae08504636b3c32e83a5c4deb80488cf15a7..b57de0895bdceb519ca82e6ad416cabfa00f6ef6 100644 --- a/optimized/DIHARD III/ls_eend_dih3_500ms.mlmodelc/analytics/coremldata.bin +++ b/optimized/DIHARD III/ls_eend_dih3_500ms.mlmodelc/analytics/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c2c1b72153067557ba269f605e6d5436836aaec8222b1c610bcb7a94471c36a8 +oid sha256:2b0fbc0b15912638e1b82cd3fe1b2566e6165a8976d68dd1d33fefa9e9447175 size 243 diff --git a/optimized/DIHARD III/ls_eend_dih3_500ms.mlmodelc/coremldata.bin b/optimized/DIHARD III/ls_eend_dih3_500ms.mlmodelc/coremldata.bin index 9043467c2467a517c25b309aafeb7e47f2c1ca63..637fbd7909859c17e1a8accf6aa5ba7d6465952b 100644 --- a/optimized/DIHARD III/ls_eend_dih3_500ms.mlmodelc/coremldata.bin +++ b/optimized/DIHARD III/ls_eend_dih3_500ms.mlmodelc/coremldata.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:03c87534e76f6c796df2fd79158fce7a8cb8952bd6fe4d2f0b28b511ebb7043c -size 1268 +oid sha256:c93174a8fe05f97fbce9af0e70529cede5d6e96857075395114abaa3c7cb0836 +size 1359 diff --git a/optimized/DIHARD III/ls_eend_dih3_500ms.mlmodelc/metadata.json b/optimized/DIHARD III/ls_eend_dih3_500ms.mlmodelc/metadata.json index 18000ccc8618118c2be60a41387f3495586dc7e4..e211d84085ab75610e1ac96fa5db7e4d79b71577 100644 --- a/optimized/DIHARD III/ls_eend_dih3_500ms.mlmodelc/metadata.json +++ b/optimized/DIHARD III/ls_eend_dih3_500ms.mlmodelc/metadata.json @@ -96,30 +96,30 @@ "Ios17.reshape" : 108, "Ios16.softmax" : 2, "Ios17.matmul" : 29, - "Ios17.transpose" : 85, - "Split" : 20, + "Ios17.transpose" : 57, + "Split" : 4, "Ios17.expandDims" : 3, "Ios17.add" : 106, "Tile" : 21, - "Ios17.sliceByIndex" : 108, - "Ios16.sigmoid" : 21, + "Ios17.sliceByIndex" : 76, + "Ios16.sigmoid" : 5, "Ios16.reduceSum" : 1, "Ios17.squeeze" : 2, - "Ios17.layerNorm" : 49, - "Ios17.batchNorm" : 20, + "Ios17.layerNorm" : 33, + "Ios17.batchNorm" : 4, "Ios17.cast" : 40, "Ios17.sqrt" : 18, "Ios17.sub" : 40, - "Ios17.conv" : 61, + "Ios17.conv" : 13, "Ios17.realDiv" : 20, "Ios17.linear" : 56, "Ios17.concat" : 26, "Ios17.clip" : 46, "Ios17.greaterEqual" : 20, - "Ios16.silu" : 34, + "Ios16.silu" : 18, "Stack" : 6, "Ios16.cumsum" : 1, - "Ios17.mul" : 143 + "Ios17.mul" : 127 }, "computePrecision" : "Mixed (Float32, Int32)", "isUpdatable" : "0", @@ -231,7 +231,7 @@ ], "userDefinedMetadata" : { "com.github.apple.coremltools.conversion_date" : "2026-04-16", - "config" : "{\"model_name\": \"dih3\", \"model_label\": \"DIHARD III\", \"variant\": \"pipeline\", \"T\": 5, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", + "config" : "{\"model_name\": \"dih3\", \"model_label\": \"DIHARD III\", \"variant\": \"pipeline\", \"chunk_size\": 5, \"step_duration_ms\": 500, \"frame_duration_ms\": 100, \"frame_duration_seconds\": 0.1, \"max_speakers\": 10, \"max_nspks\": 12, \"n_units\": 256, \"n_heads\": 4, \"enc_n_layers\": 4, \"dec_n_layers\": 2, \"conv_kernel_size\": 16, \"conv_delay\": 9, \"sample_rate\": 8000, \"win_length\": 200, \"hop_length\": 80, \"n_mels\": 23, \"context_size\": 7, \"subsampling\": 10, \"feat_type\": \"logmel23_cummn\"}", "com.github.apple.coremltools.source" : "torch==2.6.0", "com.github.apple.coremltools.version" : "9.0", "com.github.apple.coremltools.source_dialect" : "TorchScript" diff --git a/optimized/DIHARD III/ls_eend_dih3_500ms.mlmodelc/model.mil b/optimized/DIHARD III/ls_eend_dih3_500ms.mlmodelc/model.mil index d4fb1d80c3e733d6b9ac22e812c68c6db0697bc2..b13035ac5720520676a47ddea8a64f614f91249d 100644 --- a/optimized/DIHARD III/ls_eend_dih3_500ms.mlmodelc/model.mil +++ b/optimized/DIHARD III/ls_eend_dih3_500ms.mlmodelc/model.mil @@ -217,6 +217,7 @@ program(1.0) tensor var_22 = const()[name = tensor("op_22"), val = tensor(0x0p+0)]; tensor var_25 = const()[name = tensor("op_25"), val = tensor(0x1.0c6f7ap-20)]; tensor var_28 = const()[name = tensor("op_28"), val = tensor(2)]; + tensor var_31 = const()[name = tensor("op_31"), val = tensor(0)]; tensor var_34 = const()[name = tensor("op_34"), val = tensor(1)]; tensor var_36 = const()[name = tensor("op_36"), val = tensor(0x1.4f8b58p-17)]; tensor var_37 = const()[name = tensor("op_37"), val = tensor(true)]; @@ -263,8 +264,8 @@ program(1.0) tensor sqrt_s_t_1 = sqrt(x = s_t_1)[name = tensor("sqrt_s_t_1")]; tensor qk_1_transpose_x_1 = const()[name = tensor("qk_1_transpose_x_1"), val = tensor(false)]; tensor qk_1_transpose_y_1 = const()[name = tensor("qk_1_transpose_y_1"), val = tensor(true)]; - tensor k_1 = transpose(perm = k_1_perm_0, x = var_181)[name = tensor("transpose_85")]; - tensor q_1 = transpose(perm = q_1_perm_0, x = var_173)[name = tensor("transpose_86")]; + tensor k_1 = transpose(perm = k_1_perm_0, x = var_181)[name = tensor("transpose_57")]; + tensor q_1 = transpose(perm = q_1_perm_0, x = var_173)[name = tensor("transpose_58")]; tensor qk_1 = matmul(transpose_x = qk_1_transpose_x_1, transpose_y = qk_1_transpose_y_1, x = q_1, y = k_1)[name = tensor("qk_1")]; tensor var_197 = const()[name = tensor("op_197"), val = tensor([5, 1])]; tensor var_198 = reshape(shape = var_197, x = sqrt_s_t_1)[name = tensor("op_198")]; @@ -272,7 +273,7 @@ program(1.0) tensor var_200 = mul(x = qk_1, y = M_1)[name = tensor("op_200")]; tensor inner_1_transpose_x_0 = const()[name = tensor("inner_1_transpose_x_0"), val = tensor(false)]; tensor inner_1_transpose_y_0 = const()[name = tensor("inner_1_transpose_y_0"), val = tensor(false)]; - tensor v_1 = transpose(perm = v_1_perm_0, x = var_187)[name = tensor("transpose_84")]; + tensor v_1 = transpose(perm = v_1_perm_0, x = var_187)[name = tensor("transpose_56")]; tensor inner_1 = matmul(transpose_x = inner_1_transpose_x_0, transpose_y = inner_1_transpose_y_0, x = var_200, y = v_1)[name = tensor("inner_1")]; tensor var_202_transpose_x_0 = const()[name = tensor("op_202_transpose_x_0"), val = tensor(false)]; tensor var_202_transpose_y_0 = const()[name = tensor("op_202_transpose_y_0"), val = tensor(false)]; @@ -293,7 +294,7 @@ program(1.0) tensor var_215 = real_div(x = new_kv_unnorm_1, y = var_214)[name = tensor("op_215")]; tensor var_216_perm_0 = const()[name = tensor("op_216_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_3_axes_0 = const()[name = tensor("out_3_axes_0"), val = tensor([-1])]; - tensor var_216 = transpose(perm = var_216_perm_0, x = out_1)[name = tensor("transpose_83")]; + tensor var_216 = transpose(perm = var_216_perm_0, x = out_1)[name = tensor("transpose_55")]; tensor out_3 = layer_norm(axes = out_3_axes_0, epsilon = var_25, x = var_216)[name = tensor("out_3")]; tensor var_220 = const()[name = tensor("op_220"), val = tensor([1, 5, 256])]; tensor out_5 = reshape(shape = var_220, x = out_3)[name = tensor("out_5")]; @@ -341,364 +342,210 @@ program(1.0) tensor var_253 = sub(x = var_18, y = use_roll_1)[name = tensor("op_253")]; tensor var_254 = mul(x = filled_window_1, y = var_253)[name = tensor("op_254")]; tensor var_255 = mul(x = rolled_1, y = use_roll_1)[name = tensor("op_255")]; - tensor input_21 = add(x = var_254, y = var_255)[name = tensor("input_21")]; + tensor window_3 = add(x = var_254, y = var_255)[name = tensor("window_3")]; tensor var_257 = const()[name = tensor("op_257"), val = tensor(0x1p+0)]; tensor var_258 = add(x = filled_1, y = var_257)[name = tensor("op_258")]; tensor clip_1 = clip(alpha = var_22, beta = var_17, x = var_258)[name = tensor("clip_1")]; + tensor var_261_begin_0 = const()[name = tensor("op_261_begin_0"), val = tensor([0, 1, 0])]; + tensor var_261_end_0 = const()[name = tensor("op_261_end_0"), val = tensor([1, 2, 256])]; + tensor var_261_end_mask_0 = const()[name = tensor("op_261_end_mask_0"), val = tensor([true, false, true])]; + tensor var_261 = slice_by_index(begin = var_261_begin_0, end = var_261_end_0, end_mask = var_261_end_mask_0, x = x_3)[name = tensor("op_261")]; + tensor var_264_begin_0 = const()[name = tensor("op_264_begin_0"), val = tensor([0, 1, 0])]; + tensor var_264_end_0 = const()[name = tensor("op_264_end_0"), val = tensor([1, 16, 256])]; + tensor var_264_end_mask_0 = const()[name = tensor("op_264_end_mask_0"), val = tensor([true, true, true])]; + tensor var_264 = slice_by_index(begin = var_264_begin_0, end = var_264_end_0, end_mask = var_264_end_mask_0, x = window_3)[name = tensor("op_264")]; + tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; + tensor rolled_3 = concat(axis = var_34, interleave = rolled_3_interleave_0, values = (var_264, var_261))[name = tensor("rolled_3")]; + tensor clip_2 = clip(alpha = var_22, beta = var_21, x = clip_1)[name = tensor("clip_2")]; + tensor var_269 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_269")]; + tensor var_270_dtype_0 = const()[name = tensor("op_270_dtype_0"), val = tensor("fp32")]; + tensor var_271 = const()[name = tensor("op_271"), val = tensor([1, 16, 1])]; + tensor var_270 = cast(dtype = var_270_dtype_0, x = var_269)[name = tensor("cast_189")]; + tensor fill_mask_3 = reshape(shape = var_271, x = var_270)[name = tensor("fill_mask_3")]; + tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_261)[name = tensor("x_expanded_3")]; + tensor var_275 = sub(x = var_18, y = fill_mask_3)[name = tensor("op_275")]; + tensor var_276 = mul(x = window_3, y = var_275)[name = tensor("op_276")]; + tensor var_277 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_277")]; + tensor filled_window_3 = add(x = var_276, y = var_277)[name = tensor("filled_window_3")]; + tensor var_279 = greater_equal(x = clip_1, y = var_17)[name = tensor("op_279")]; + tensor var_280_dtype_0 = const()[name = tensor("op_280_dtype_0"), val = tensor("fp32")]; + tensor var_281 = const()[name = tensor("op_281"), val = tensor([1, 1, 1])]; + tensor var_280 = cast(dtype = var_280_dtype_0, x = var_279)[name = tensor("cast_188")]; + tensor use_roll_3 = reshape(shape = var_281, x = var_280)[name = tensor("use_roll_3")]; + tensor var_283 = sub(x = var_18, y = use_roll_3)[name = tensor("op_283")]; + tensor var_284 = mul(x = filled_window_3, y = var_283)[name = tensor("op_284")]; + tensor var_285 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_285")]; + tensor window_5 = add(x = var_284, y = var_285)[name = tensor("window_5")]; + tensor var_287 = const()[name = tensor("op_287"), val = tensor(0x1p+0)]; + tensor var_288 = add(x = clip_1, y = var_287)[name = tensor("op_288")]; + tensor clip_3 = clip(alpha = var_22, beta = var_17, x = var_288)[name = tensor("clip_3")]; + tensor var_291_begin_0 = const()[name = tensor("op_291_begin_0"), val = tensor([0, 2, 0])]; + tensor var_291_end_0 = const()[name = tensor("op_291_end_0"), val = tensor([1, 3, 256])]; + tensor var_291_end_mask_0 = const()[name = tensor("op_291_end_mask_0"), val = tensor([true, false, true])]; + tensor var_291 = slice_by_index(begin = var_291_begin_0, end = var_291_end_0, end_mask = var_291_end_mask_0, x = x_3)[name = tensor("op_291")]; + tensor var_294_begin_0 = const()[name = tensor("op_294_begin_0"), val = tensor([0, 1, 0])]; + tensor var_294_end_0 = const()[name = tensor("op_294_end_0"), val = tensor([1, 16, 256])]; + tensor var_294_end_mask_0 = const()[name = tensor("op_294_end_mask_0"), val = tensor([true, true, true])]; + tensor var_294 = slice_by_index(begin = var_294_begin_0, end = var_294_end_0, end_mask = var_294_end_mask_0, x = window_5)[name = tensor("op_294")]; + tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; + tensor rolled_5 = concat(axis = var_34, interleave = rolled_5_interleave_0, values = (var_294, var_291))[name = tensor("rolled_5")]; + tensor clip_4 = clip(alpha = var_22, beta = var_21, x = clip_3)[name = tensor("clip_4")]; + tensor var_299 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_299")]; + tensor var_300_dtype_0 = const()[name = tensor("op_300_dtype_0"), val = tensor("fp32")]; + tensor var_301 = const()[name = tensor("op_301"), val = tensor([1, 16, 1])]; + tensor var_300 = cast(dtype = var_300_dtype_0, x = var_299)[name = tensor("cast_187")]; + tensor fill_mask_5 = reshape(shape = var_301, x = var_300)[name = tensor("fill_mask_5")]; + tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_291)[name = tensor("x_expanded_5")]; + tensor var_305 = sub(x = var_18, y = fill_mask_5)[name = tensor("op_305")]; + tensor var_306 = mul(x = window_5, y = var_305)[name = tensor("op_306")]; + tensor var_307 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_307")]; + tensor filled_window_5 = add(x = var_306, y = var_307)[name = tensor("filled_window_5")]; + tensor var_309 = greater_equal(x = clip_3, y = var_17)[name = tensor("op_309")]; + tensor var_310_dtype_0 = const()[name = tensor("op_310_dtype_0"), val = tensor("fp32")]; + tensor var_311 = const()[name = tensor("op_311"), val = tensor([1, 1, 1])]; + tensor var_310 = cast(dtype = var_310_dtype_0, x = var_309)[name = tensor("cast_186")]; + tensor use_roll_5 = reshape(shape = var_311, x = var_310)[name = tensor("use_roll_5")]; + tensor var_313 = sub(x = var_18, y = use_roll_5)[name = tensor("op_313")]; + tensor var_314 = mul(x = filled_window_5, y = var_313)[name = tensor("op_314")]; + tensor var_315 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_315")]; + tensor window_7 = add(x = var_314, y = var_315)[name = tensor("window_7")]; + tensor var_317 = const()[name = tensor("op_317"), val = tensor(0x1p+0)]; + tensor var_318 = add(x = clip_3, y = var_317)[name = tensor("op_318")]; + tensor clip_5 = clip(alpha = var_22, beta = var_17, x = var_318)[name = tensor("clip_5")]; + tensor var_321_begin_0 = const()[name = tensor("op_321_begin_0"), val = tensor([0, 3, 0])]; + tensor var_321_end_0 = const()[name = tensor("op_321_end_0"), val = tensor([1, 4, 256])]; + tensor var_321_end_mask_0 = const()[name = tensor("op_321_end_mask_0"), val = tensor([true, false, true])]; + tensor var_321 = slice_by_index(begin = var_321_begin_0, end = var_321_end_0, end_mask = var_321_end_mask_0, x = x_3)[name = tensor("op_321")]; + tensor var_324_begin_0 = const()[name = tensor("op_324_begin_0"), val = tensor([0, 1, 0])]; + tensor var_324_end_0 = const()[name = tensor("op_324_end_0"), val = tensor([1, 16, 256])]; + tensor var_324_end_mask_0 = const()[name = tensor("op_324_end_mask_0"), val = tensor([true, true, true])]; + tensor var_324 = slice_by_index(begin = var_324_begin_0, end = var_324_end_0, end_mask = var_324_end_mask_0, x = window_7)[name = tensor("op_324")]; + tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; + tensor rolled_7 = concat(axis = var_34, interleave = rolled_7_interleave_0, values = (var_324, var_321))[name = tensor("rolled_7")]; + tensor clip_6 = clip(alpha = var_22, beta = var_21, x = clip_5)[name = tensor("clip_6")]; + tensor var_329 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_329")]; + tensor var_330_dtype_0 = const()[name = tensor("op_330_dtype_0"), val = tensor("fp32")]; + tensor var_331 = const()[name = tensor("op_331"), val = tensor([1, 16, 1])]; + tensor var_330 = cast(dtype = var_330_dtype_0, x = var_329)[name = tensor("cast_185")]; + tensor fill_mask_7 = reshape(shape = var_331, x = var_330)[name = tensor("fill_mask_7")]; + tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_321)[name = tensor("x_expanded_7")]; + tensor var_335 = sub(x = var_18, y = fill_mask_7)[name = tensor("op_335")]; + tensor var_336 = mul(x = window_7, y = var_335)[name = tensor("op_336")]; + tensor var_337 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_337")]; + tensor filled_window_7 = add(x = var_336, y = var_337)[name = tensor("filled_window_7")]; + tensor var_339 = greater_equal(x = clip_5, y = var_17)[name = tensor("op_339")]; + tensor var_340_dtype_0 = const()[name = tensor("op_340_dtype_0"), val = tensor("fp32")]; + tensor var_341 = const()[name = tensor("op_341"), val = tensor([1, 1, 1])]; + tensor var_340 = cast(dtype = var_340_dtype_0, x = var_339)[name = tensor("cast_184")]; + tensor use_roll_7 = reshape(shape = var_341, x = var_340)[name = tensor("use_roll_7")]; + tensor var_343 = sub(x = var_18, y = use_roll_7)[name = tensor("op_343")]; + tensor var_344 = mul(x = filled_window_7, y = var_343)[name = tensor("op_344")]; + tensor var_345 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_345")]; + tensor window_9 = add(x = var_344, y = var_345)[name = tensor("window_9")]; + tensor var_347 = const()[name = tensor("op_347"), val = tensor(0x1p+0)]; + tensor var_348 = add(x = clip_5, y = var_347)[name = tensor("op_348")]; + tensor clip_7 = clip(alpha = var_22, beta = var_17, x = var_348)[name = tensor("clip_7")]; + tensor var_351_begin_0 = const()[name = tensor("op_351_begin_0"), val = tensor([0, 4, 0])]; + tensor var_351_end_0 = const()[name = tensor("op_351_end_0"), val = tensor([1, 1, 256])]; + tensor var_351_end_mask_0 = const()[name = tensor("op_351_end_mask_0"), val = tensor([true, true, true])]; + tensor var_351 = slice_by_index(begin = var_351_begin_0, end = var_351_end_0, end_mask = var_351_end_mask_0, x = x_3)[name = tensor("op_351")]; + tensor var_354_begin_0 = const()[name = tensor("op_354_begin_0"), val = tensor([0, 1, 0])]; + tensor var_354_end_0 = const()[name = tensor("op_354_end_0"), val = tensor([1, 16, 256])]; + tensor var_354_end_mask_0 = const()[name = tensor("op_354_end_mask_0"), val = tensor([true, true, true])]; + tensor var_354 = slice_by_index(begin = var_354_begin_0, end = var_354_end_0, end_mask = var_354_end_mask_0, x = window_9)[name = tensor("op_354")]; + tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; + tensor rolled_9 = concat(axis = var_34, interleave = rolled_9_interleave_0, values = (var_354, var_351))[name = tensor("rolled_9")]; + tensor clip_8 = clip(alpha = var_22, beta = var_21, x = clip_7)[name = tensor("clip_8")]; + tensor var_359 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_359")]; + tensor var_360_dtype_0 = const()[name = tensor("op_360_dtype_0"), val = tensor("fp32")]; + tensor var_361 = const()[name = tensor("op_361"), val = tensor([1, 16, 1])]; + tensor var_360 = cast(dtype = var_360_dtype_0, x = var_359)[name = tensor("cast_183")]; + tensor fill_mask_9 = reshape(shape = var_361, x = var_360)[name = tensor("fill_mask_9")]; + tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; + tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_351)[name = tensor("x_expanded_9")]; + tensor var_365 = sub(x = var_18, y = fill_mask_9)[name = tensor("op_365")]; + tensor var_366 = mul(x = window_9, y = var_365)[name = tensor("op_366")]; + tensor var_367 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_367")]; + tensor filled_window_9 = add(x = var_366, y = var_367)[name = tensor("filled_window_9")]; + tensor var_369 = greater_equal(x = clip_7, y = var_17)[name = tensor("op_369")]; + tensor var_370_dtype_0 = const()[name = tensor("op_370_dtype_0"), val = tensor("fp32")]; + tensor var_371 = const()[name = tensor("op_371"), val = tensor([1, 1, 1])]; + tensor var_370 = cast(dtype = var_370_dtype_0, x = var_369)[name = tensor("cast_182")]; + tensor use_roll_9 = reshape(shape = var_371, x = var_370)[name = tensor("use_roll_9")]; + tensor var_373 = sub(x = var_18, y = use_roll_9)[name = tensor("op_373")]; + tensor var_374 = mul(x = filled_window_9, y = var_373)[name = tensor("op_374")]; + tensor var_375 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_375")]; + tensor window_11 = add(x = var_374, y = var_375)[name = tensor("window_11")]; + tensor var_377 = const()[name = tensor("op_377"), val = tensor(0x1p+0)]; + tensor var_378 = add(x = clip_7, y = var_377)[name = tensor("op_378")]; + tensor clip_9 = clip(alpha = var_22, beta = var_17, x = var_378)[name = tensor("clip_9")]; + tensor input_21_interleave_0 = const()[name = tensor("input_21_interleave_0"), val = tensor(false)]; + tensor input_21 = concat(axis = var_31, interleave = input_21_interleave_0, values = (window_3, window_5, window_7, window_9, window_11))[name = tensor("input_21")]; tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; - tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; + tensor x_5 = layer_norm(axes = x_5_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_21)[name = tensor("x_5")]; tensor input_23_perm_0 = const()[name = tensor("input_23_perm_0"), val = tensor([0, 2, 1])]; tensor inputs_3_pad_type_0 = const()[name = tensor("inputs_3_pad_type_0"), val = tensor("valid")]; tensor inputs_3_strides_0 = const()[name = tensor("inputs_3_strides_0"), val = tensor([1])]; tensor inputs_3_pad_0 = const()[name = tensor("inputs_3_pad_0"), val = tensor([0, 0])]; tensor inputs_3_dilations_0 = const()[name = tensor("inputs_3_dilations_0"), val = tensor([1])]; tensor inputs_3_groups_0 = const()[name = tensor("inputs_3_groups_0"), val = tensor(1)]; - tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_82")]; - tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; - tensor var_279_split_sizes_0 = const()[name = tensor("op_279_split_sizes_0"), val = tensor([256, 256])]; - tensor var_279_axis_0 = const()[name = tensor("op_279_axis_0"), val = tensor(1)]; - tensor var_279_0, tensor var_279_1 = split(axis = var_279_axis_0, split_sizes = var_279_split_sizes_0, x = inputs_3)[name = tensor("op_279")]; - tensor var_281 = sigmoid(x = var_279_1)[name = tensor("op_281")]; - tensor inputs_5 = mul(x = var_279_0, y = var_281)[name = tensor("inputs_5")]; + tensor input_23 = transpose(perm = input_23_perm_0, x = x_5)[name = tensor("transpose_54")]; + tensor inputs_3 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_3_dilations_0, groups = inputs_3_groups_0, pad = inputs_3_pad_0, pad_type = inputs_3_pad_type_0, strides = inputs_3_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_23)[name = tensor("inputs_3")]; + tensor var_401_split_sizes_0 = const()[name = tensor("op_401_split_sizes_0"), val = tensor([256, 256])]; + tensor var_401_axis_0 = const()[name = tensor("op_401_axis_0"), val = tensor(1)]; + tensor var_401_0, tensor var_401_1 = split(axis = var_401_axis_0, split_sizes = var_401_split_sizes_0, x = inputs_3)[name = tensor("op_401")]; + tensor var_403 = sigmoid(x = var_401_1)[name = tensor("op_403")]; + tensor inputs_5 = mul(x = var_401_0, y = var_403)[name = tensor("inputs_5")]; tensor outputs_aug_1_pad_type_0 = const()[name = tensor("outputs_aug_1_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_1_pad_0 = const()[name = tensor("outputs_aug_1_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_1_groups_0 = const()[name = tensor("outputs_aug_1_groups_0"), val = tensor(256)]; tensor outputs_aug_1_strides_0 = const()[name = tensor("outputs_aug_1_strides_0"), val = tensor([1])]; tensor outputs_aug_1_dilations_0 = const()[name = tensor("outputs_aug_1_dilations_0"), val = tensor([1])]; - tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; + tensor outputs_aug_1 = conv(dilations = outputs_aug_1_dilations_0, groups = outputs_aug_1_groups_0, pad = outputs_aug_1_pad_0, pad_type = outputs_aug_1_pad_type_0, strides = outputs_aug_1_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_5)[name = tensor("outputs_aug_1")]; tensor input_25_begin_0 = const()[name = tensor("input_25_begin_0"), val = tensor([0, 0, 0])]; - tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([1, 256, 16])]; + tensor input_25_end_0 = const()[name = tensor("input_25_end_0"), val = tensor([5, 256, 16])]; tensor input_25_end_mask_0 = const()[name = tensor("input_25_end_mask_0"), val = tensor([true, true, false])]; - tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; - tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; - tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; + tensor input_25 = slice_by_index(begin = input_25_begin_0, end = input_25_end_0, end_mask = input_25_end_mask_0, x = outputs_aug_1)[name = tensor("input_25")]; + tensor inputs_7 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_25)[name = tensor("inputs_7")]; + tensor input_27 = silu(x = inputs_7)[name = tensor("input_27")]; tensor input_29_pad_type_0 = const()[name = tensor("input_29_pad_type_0"), val = tensor("valid")]; tensor input_29_strides_0 = const()[name = tensor("input_29_strides_0"), val = tensor([1])]; tensor input_29_pad_0 = const()[name = tensor("input_29_pad_0"), val = tensor([0, 0])]; tensor input_29_dilations_0 = const()[name = tensor("input_29_dilations_0"), val = tensor([1])]; tensor input_29_groups_0 = const()[name = tensor("input_29_groups_0"), val = tensor(1)]; - tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; + tensor input_29 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_29_dilations_0, groups = input_29_groups_0, pad = input_29_pad_0, pad_type = input_29_pad_type_0, strides = input_29_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_27)[name = tensor("input_29")]; tensor conv_out_1_perm_0 = const()[name = tensor("conv_out_1_perm_0"), val = tensor([0, 2, 1])]; - tensor var_312_begin_0 = const()[name = tensor("op_312_begin_0"), val = tensor([0, -1, 0])]; - tensor var_312_end_0 = const()[name = tensor("op_312_end_0"), val = tensor([1, 16, 256])]; - tensor var_312_end_mask_0 = const()[name = tensor("op_312_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_81")]; - tensor var_312 = slice_by_index(begin = var_312_begin_0, end = var_312_end_0, end_mask = var_312_end_mask_0, x = conv_out_1)[name = tensor("op_312")]; - tensor var_315_begin_0 = const()[name = tensor("op_315_begin_0"), val = tensor([0, 1, 0])]; - tensor var_315_end_0 = const()[name = tensor("op_315_end_0"), val = tensor([1, 2, 256])]; - tensor var_315_end_mask_0 = const()[name = tensor("op_315_end_mask_0"), val = tensor([true, false, true])]; - tensor var_315 = slice_by_index(begin = var_315_begin_0, end = var_315_end_0, end_mask = var_315_end_mask_0, x = x_3)[name = tensor("op_315")]; - tensor var_318_begin_0 = const()[name = tensor("op_318_begin_0"), val = tensor([0, 1, 0])]; - tensor var_318_end_0 = const()[name = tensor("op_318_end_0"), val = tensor([1, 16, 256])]; - tensor var_318_end_mask_0 = const()[name = tensor("op_318_end_mask_0"), val = tensor([true, true, true])]; - tensor var_318 = slice_by_index(begin = var_318_begin_0, end = var_318_end_0, end_mask = var_318_end_mask_0, x = input_21)[name = tensor("op_318")]; - tensor rolled_3_interleave_0 = const()[name = tensor("rolled_3_interleave_0"), val = tensor(false)]; - tensor rolled_3 = concat(axis = var_34, interleave = rolled_3_interleave_0, values = (var_318, var_315))[name = tensor("rolled_3")]; - tensor clip_2 = clip(alpha = var_22, beta = var_21, x = clip_1)[name = tensor("clip_2")]; - tensor var_323 = equal(x = encoder__conv_positions, y = clip_2)[name = tensor("op_323")]; - tensor var_324_dtype_0 = const()[name = tensor("op_324_dtype_0"), val = tensor("fp32")]; - tensor var_325 = const()[name = tensor("op_325"), val = tensor([1, 16, 1])]; - tensor var_324 = cast(dtype = var_324_dtype_0, x = var_323)[name = tensor("cast_189")]; - tensor fill_mask_3 = reshape(shape = var_325, x = var_324)[name = tensor("fill_mask_3")]; - tensor x_expanded_3_reps_0 = const()[name = tensor("x_expanded_3_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_3 = tile(reps = x_expanded_3_reps_0, x = var_315)[name = tensor("x_expanded_3")]; - tensor var_329 = sub(x = var_18, y = fill_mask_3)[name = tensor("op_329")]; - tensor var_330 = mul(x = input_21, y = var_329)[name = tensor("op_330")]; - tensor var_331 = mul(x = x_expanded_3, y = fill_mask_3)[name = tensor("op_331")]; - tensor filled_window_3 = add(x = var_330, y = var_331)[name = tensor("filled_window_3")]; - tensor var_333 = greater_equal(x = clip_1, y = var_17)[name = tensor("op_333")]; - tensor var_334_dtype_0 = const()[name = tensor("op_334_dtype_0"), val = tensor("fp32")]; - tensor var_335 = const()[name = tensor("op_335"), val = tensor([1, 1, 1])]; - tensor var_334 = cast(dtype = var_334_dtype_0, x = var_333)[name = tensor("cast_188")]; - tensor use_roll_3 = reshape(shape = var_335, x = var_334)[name = tensor("use_roll_3")]; - tensor var_337 = sub(x = var_18, y = use_roll_3)[name = tensor("op_337")]; - tensor var_338 = mul(x = filled_window_3, y = var_337)[name = tensor("op_338")]; - tensor var_339 = mul(x = rolled_3, y = use_roll_3)[name = tensor("op_339")]; - tensor input_31 = add(x = var_338, y = var_339)[name = tensor("input_31")]; - tensor var_341 = const()[name = tensor("op_341"), val = tensor(0x1p+0)]; - tensor var_342 = add(x = clip_1, y = var_341)[name = tensor("op_342")]; - tensor clip_3 = clip(alpha = var_22, beta = var_17, x = var_342)[name = tensor("clip_3")]; + tensor var_434_begin_0 = const()[name = tensor("op_434_begin_0"), val = tensor([0, -1, 0])]; + tensor var_434_end_0 = const()[name = tensor("op_434_end_0"), val = tensor([5, 16, 256])]; + tensor var_434_end_mask_0 = const()[name = tensor("op_434_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_1 = transpose(perm = conv_out_1_perm_0, x = input_29)[name = tensor("transpose_53")]; + tensor var_434 = slice_by_index(begin = var_434_begin_0, end = var_434_end_0, end_mask = var_434_end_mask_0, x = conv_out_1)[name = tensor("op_434")]; + tensor var_436_perm_0 = const()[name = tensor("op_436_perm_0"), val = tensor([1, 0, 2])]; + tensor var_436 = transpose(perm = var_436_perm_0, x = var_434)[name = tensor("transpose_52")]; + tensor input_31 = add(x = x_3, y = var_436)[name = tensor("input_31")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor input_33 = layer_norm(axes = input_33_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_31)[name = tensor("input_33")]; + tensor inputs_9 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_33)[name = tensor("linear_8")]; + tensor input_35 = silu(x = inputs_9)[name = tensor("input_35")]; + tensor input_39 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_35)[name = tensor("linear_9")]; + tensor var_459 = const()[name = tensor("op_459"), val = tensor(0x1p-1)]; + tensor var_460 = mul(x = input_39, y = var_459)[name = tensor("op_460")]; + tensor input_41 = add(x = var_460, y = input_31)[name = tensor("input_41")]; + tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; + tensor input_43 = layer_norm(axes = input_43_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_36, gamma = encoder_layer_norm_0_weight, x = input_41)[name = tensor("input_43")]; + tensor input_45_axes_0 = const()[name = tensor("input_45_axes_0"), val = tensor([-1])]; + tensor input_45 = layer_norm(axes = input_45_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_43)[name = tensor("input_45")]; + tensor inputs_11 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_45)[name = tensor("linear_10")]; + tensor input_47 = silu(x = inputs_11)[name = tensor("input_47")]; + tensor input_51 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_47)[name = tensor("linear_11")]; + tensor var_489 = const()[name = tensor("op_489"), val = tensor(0x1p-1)]; + tensor var_490 = mul(x = input_51, y = var_489)[name = tensor("op_490")]; + tensor input_53 = add(x = var_490, y = input_43)[name = tensor("input_53")]; tensor x_7_axes_0 = const()[name = tensor("x_7_axes_0"), val = tensor([-1])]; - tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_31)[name = tensor("x_7")]; - tensor input_33_perm_0 = const()[name = tensor("input_33_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_9_pad_type_0 = const()[name = tensor("inputs_9_pad_type_0"), val = tensor("valid")]; - tensor inputs_9_strides_0 = const()[name = tensor("inputs_9_strides_0"), val = tensor([1])]; - tensor inputs_9_pad_0 = const()[name = tensor("inputs_9_pad_0"), val = tensor([0, 0])]; - tensor inputs_9_dilations_0 = const()[name = tensor("inputs_9_dilations_0"), val = tensor([1])]; - tensor inputs_9_groups_0 = const()[name = tensor("inputs_9_groups_0"), val = tensor(1)]; - tensor input_33 = transpose(perm = input_33_perm_0, x = x_7)[name = tensor("transpose_80")]; - tensor inputs_9 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_9_dilations_0, groups = inputs_9_groups_0, pad = inputs_9_pad_0, pad_type = inputs_9_pad_type_0, strides = inputs_9_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_33)[name = tensor("inputs_9")]; - tensor var_363_split_sizes_0 = const()[name = tensor("op_363_split_sizes_0"), val = tensor([256, 256])]; - tensor var_363_axis_0 = const()[name = tensor("op_363_axis_0"), val = tensor(1)]; - tensor var_363_0, tensor var_363_1 = split(axis = var_363_axis_0, split_sizes = var_363_split_sizes_0, x = inputs_9)[name = tensor("op_363")]; - tensor var_365 = sigmoid(x = var_363_1)[name = tensor("op_365")]; - tensor inputs_11 = mul(x = var_363_0, y = var_365)[name = tensor("inputs_11")]; - tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; - tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; - tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; - tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_11)[name = tensor("outputs_aug_3")]; - tensor input_35_begin_0 = const()[name = tensor("input_35_begin_0"), val = tensor([0, 0, 0])]; - tensor input_35_end_0 = const()[name = tensor("input_35_end_0"), val = tensor([1, 256, 16])]; - tensor input_35_end_mask_0 = const()[name = tensor("input_35_end_mask_0"), val = tensor([true, true, false])]; - tensor input_35 = slice_by_index(begin = input_35_begin_0, end = input_35_end_0, end_mask = input_35_end_mask_0, x = outputs_aug_3)[name = tensor("input_35")]; - tensor inputs_13 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_35)[name = tensor("inputs_13")]; - tensor input_37 = silu(x = inputs_13)[name = tensor("input_37")]; - tensor input_39_pad_type_0 = const()[name = tensor("input_39_pad_type_0"), val = tensor("valid")]; - tensor input_39_strides_0 = const()[name = tensor("input_39_strides_0"), val = tensor([1])]; - tensor input_39_pad_0 = const()[name = tensor("input_39_pad_0"), val = tensor([0, 0])]; - tensor input_39_dilations_0 = const()[name = tensor("input_39_dilations_0"), val = tensor([1])]; - tensor input_39_groups_0 = const()[name = tensor("input_39_groups_0"), val = tensor(1)]; - tensor input_39 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_37)[name = tensor("input_39")]; - tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; - tensor var_396_begin_0 = const()[name = tensor("op_396_begin_0"), val = tensor([0, -1, 0])]; - tensor var_396_end_0 = const()[name = tensor("op_396_end_0"), val = tensor([1, 16, 256])]; - tensor var_396_end_mask_0 = const()[name = tensor("op_396_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_39)[name = tensor("transpose_79")]; - tensor var_396 = slice_by_index(begin = var_396_begin_0, end = var_396_end_0, end_mask = var_396_end_mask_0, x = conv_out_3)[name = tensor("op_396")]; - tensor var_399_begin_0 = const()[name = tensor("op_399_begin_0"), val = tensor([0, 2, 0])]; - tensor var_399_end_0 = const()[name = tensor("op_399_end_0"), val = tensor([1, 3, 256])]; - tensor var_399_end_mask_0 = const()[name = tensor("op_399_end_mask_0"), val = tensor([true, false, true])]; - tensor var_399 = slice_by_index(begin = var_399_begin_0, end = var_399_end_0, end_mask = var_399_end_mask_0, x = x_3)[name = tensor("op_399")]; - tensor var_402_begin_0 = const()[name = tensor("op_402_begin_0"), val = tensor([0, 1, 0])]; - tensor var_402_end_0 = const()[name = tensor("op_402_end_0"), val = tensor([1, 16, 256])]; - tensor var_402_end_mask_0 = const()[name = tensor("op_402_end_mask_0"), val = tensor([true, true, true])]; - tensor var_402 = slice_by_index(begin = var_402_begin_0, end = var_402_end_0, end_mask = var_402_end_mask_0, x = input_31)[name = tensor("op_402")]; - tensor rolled_5_interleave_0 = const()[name = tensor("rolled_5_interleave_0"), val = tensor(false)]; - tensor rolled_5 = concat(axis = var_34, interleave = rolled_5_interleave_0, values = (var_402, var_399))[name = tensor("rolled_5")]; - tensor clip_4 = clip(alpha = var_22, beta = var_21, x = clip_3)[name = tensor("clip_4")]; - tensor var_407 = equal(x = encoder__conv_positions, y = clip_4)[name = tensor("op_407")]; - tensor var_408_dtype_0 = const()[name = tensor("op_408_dtype_0"), val = tensor("fp32")]; - tensor var_409 = const()[name = tensor("op_409"), val = tensor([1, 16, 1])]; - tensor var_408 = cast(dtype = var_408_dtype_0, x = var_407)[name = tensor("cast_187")]; - tensor fill_mask_5 = reshape(shape = var_409, x = var_408)[name = tensor("fill_mask_5")]; - tensor x_expanded_5_reps_0 = const()[name = tensor("x_expanded_5_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_5 = tile(reps = x_expanded_5_reps_0, x = var_399)[name = tensor("x_expanded_5")]; - tensor var_413 = sub(x = var_18, y = fill_mask_5)[name = tensor("op_413")]; - tensor var_414 = mul(x = input_31, y = var_413)[name = tensor("op_414")]; - tensor var_415 = mul(x = x_expanded_5, y = fill_mask_5)[name = tensor("op_415")]; - tensor filled_window_5 = add(x = var_414, y = var_415)[name = tensor("filled_window_5")]; - tensor var_417 = greater_equal(x = clip_3, y = var_17)[name = tensor("op_417")]; - tensor var_418_dtype_0 = const()[name = tensor("op_418_dtype_0"), val = tensor("fp32")]; - tensor var_419 = const()[name = tensor("op_419"), val = tensor([1, 1, 1])]; - tensor var_418 = cast(dtype = var_418_dtype_0, x = var_417)[name = tensor("cast_186")]; - tensor use_roll_5 = reshape(shape = var_419, x = var_418)[name = tensor("use_roll_5")]; - tensor var_421 = sub(x = var_18, y = use_roll_5)[name = tensor("op_421")]; - tensor var_422 = mul(x = filled_window_5, y = var_421)[name = tensor("op_422")]; - tensor var_423 = mul(x = rolled_5, y = use_roll_5)[name = tensor("op_423")]; - tensor input_41 = add(x = var_422, y = var_423)[name = tensor("input_41")]; - tensor var_425 = const()[name = tensor("op_425"), val = tensor(0x1p+0)]; - tensor var_426 = add(x = clip_3, y = var_425)[name = tensor("op_426")]; - tensor clip_5 = clip(alpha = var_22, beta = var_17, x = var_426)[name = tensor("clip_5")]; - tensor x_9_axes_0 = const()[name = tensor("x_9_axes_0"), val = tensor([-1])]; - tensor x_9 = layer_norm(axes = x_9_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_41)[name = tensor("x_9")]; - tensor input_43_perm_0 = const()[name = tensor("input_43_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_15_pad_type_0 = const()[name = tensor("inputs_15_pad_type_0"), val = tensor("valid")]; - tensor inputs_15_strides_0 = const()[name = tensor("inputs_15_strides_0"), val = tensor([1])]; - tensor inputs_15_pad_0 = const()[name = tensor("inputs_15_pad_0"), val = tensor([0, 0])]; - tensor inputs_15_dilations_0 = const()[name = tensor("inputs_15_dilations_0"), val = tensor([1])]; - tensor inputs_15_groups_0 = const()[name = tensor("inputs_15_groups_0"), val = tensor(1)]; - tensor input_43 = transpose(perm = input_43_perm_0, x = x_9)[name = tensor("transpose_78")]; - tensor inputs_15 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_15_dilations_0, groups = inputs_15_groups_0, pad = inputs_15_pad_0, pad_type = inputs_15_pad_type_0, strides = inputs_15_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_43)[name = tensor("inputs_15")]; - tensor var_447_split_sizes_0 = const()[name = tensor("op_447_split_sizes_0"), val = tensor([256, 256])]; - tensor var_447_axis_0 = const()[name = tensor("op_447_axis_0"), val = tensor(1)]; - tensor var_447_0, tensor var_447_1 = split(axis = var_447_axis_0, split_sizes = var_447_split_sizes_0, x = inputs_15)[name = tensor("op_447")]; - tensor var_449 = sigmoid(x = var_447_1)[name = tensor("op_449")]; - tensor inputs_17 = mul(x = var_447_0, y = var_449)[name = tensor("inputs_17")]; - tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; - tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; - tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; - tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_17)[name = tensor("outputs_aug_5")]; - tensor input_45_begin_0 = const()[name = tensor("input_45_begin_0"), val = tensor([0, 0, 0])]; - tensor input_45_end_0 = const()[name = tensor("input_45_end_0"), val = tensor([1, 256, 16])]; - tensor input_45_end_mask_0 = const()[name = tensor("input_45_end_mask_0"), val = tensor([true, true, false])]; - tensor input_45 = slice_by_index(begin = input_45_begin_0, end = input_45_end_0, end_mask = input_45_end_mask_0, x = outputs_aug_5)[name = tensor("input_45")]; - tensor inputs_19 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_45)[name = tensor("inputs_19")]; - tensor input_47 = silu(x = inputs_19)[name = tensor("input_47")]; - tensor input_49_pad_type_0 = const()[name = tensor("input_49_pad_type_0"), val = tensor("valid")]; - tensor input_49_strides_0 = const()[name = tensor("input_49_strides_0"), val = tensor([1])]; - tensor input_49_pad_0 = const()[name = tensor("input_49_pad_0"), val = tensor([0, 0])]; - tensor input_49_dilations_0 = const()[name = tensor("input_49_dilations_0"), val = tensor([1])]; - tensor input_49_groups_0 = const()[name = tensor("input_49_groups_0"), val = tensor(1)]; - tensor input_49 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_49_dilations_0, groups = input_49_groups_0, pad = input_49_pad_0, pad_type = input_49_pad_type_0, strides = input_49_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_47)[name = tensor("input_49")]; - tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; - tensor var_480_begin_0 = const()[name = tensor("op_480_begin_0"), val = tensor([0, -1, 0])]; - tensor var_480_end_0 = const()[name = tensor("op_480_end_0"), val = tensor([1, 16, 256])]; - tensor var_480_end_mask_0 = const()[name = tensor("op_480_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_49)[name = tensor("transpose_77")]; - tensor var_480 = slice_by_index(begin = var_480_begin_0, end = var_480_end_0, end_mask = var_480_end_mask_0, x = conv_out_5)[name = tensor("op_480")]; - tensor var_483_begin_0 = const()[name = tensor("op_483_begin_0"), val = tensor([0, 3, 0])]; - tensor var_483_end_0 = const()[name = tensor("op_483_end_0"), val = tensor([1, 4, 256])]; - tensor var_483_end_mask_0 = const()[name = tensor("op_483_end_mask_0"), val = tensor([true, false, true])]; - tensor var_483 = slice_by_index(begin = var_483_begin_0, end = var_483_end_0, end_mask = var_483_end_mask_0, x = x_3)[name = tensor("op_483")]; - tensor var_486_begin_0 = const()[name = tensor("op_486_begin_0"), val = tensor([0, 1, 0])]; - tensor var_486_end_0 = const()[name = tensor("op_486_end_0"), val = tensor([1, 16, 256])]; - tensor var_486_end_mask_0 = const()[name = tensor("op_486_end_mask_0"), val = tensor([true, true, true])]; - tensor var_486 = slice_by_index(begin = var_486_begin_0, end = var_486_end_0, end_mask = var_486_end_mask_0, x = input_41)[name = tensor("op_486")]; - tensor rolled_7_interleave_0 = const()[name = tensor("rolled_7_interleave_0"), val = tensor(false)]; - tensor rolled_7 = concat(axis = var_34, interleave = rolled_7_interleave_0, values = (var_486, var_483))[name = tensor("rolled_7")]; - tensor clip_6 = clip(alpha = var_22, beta = var_21, x = clip_5)[name = tensor("clip_6")]; - tensor var_491 = equal(x = encoder__conv_positions, y = clip_6)[name = tensor("op_491")]; - tensor var_492_dtype_0 = const()[name = tensor("op_492_dtype_0"), val = tensor("fp32")]; - tensor var_493 = const()[name = tensor("op_493"), val = tensor([1, 16, 1])]; - tensor var_492 = cast(dtype = var_492_dtype_0, x = var_491)[name = tensor("cast_185")]; - tensor fill_mask_7 = reshape(shape = var_493, x = var_492)[name = tensor("fill_mask_7")]; - tensor x_expanded_7_reps_0 = const()[name = tensor("x_expanded_7_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_7 = tile(reps = x_expanded_7_reps_0, x = var_483)[name = tensor("x_expanded_7")]; - tensor var_497 = sub(x = var_18, y = fill_mask_7)[name = tensor("op_497")]; - tensor var_498 = mul(x = input_41, y = var_497)[name = tensor("op_498")]; - tensor var_499 = mul(x = x_expanded_7, y = fill_mask_7)[name = tensor("op_499")]; - tensor filled_window_7 = add(x = var_498, y = var_499)[name = tensor("filled_window_7")]; - tensor var_501 = greater_equal(x = clip_5, y = var_17)[name = tensor("op_501")]; - tensor var_502_dtype_0 = const()[name = tensor("op_502_dtype_0"), val = tensor("fp32")]; - tensor var_503 = const()[name = tensor("op_503"), val = tensor([1, 1, 1])]; - tensor var_502 = cast(dtype = var_502_dtype_0, x = var_501)[name = tensor("cast_184")]; - tensor use_roll_7 = reshape(shape = var_503, x = var_502)[name = tensor("use_roll_7")]; - tensor var_505 = sub(x = var_18, y = use_roll_7)[name = tensor("op_505")]; - tensor var_506 = mul(x = filled_window_7, y = var_505)[name = tensor("op_506")]; - tensor var_507 = mul(x = rolled_7, y = use_roll_7)[name = tensor("op_507")]; - tensor input_51 = add(x = var_506, y = var_507)[name = tensor("input_51")]; - tensor var_509 = const()[name = tensor("op_509"), val = tensor(0x1p+0)]; - tensor var_510 = add(x = clip_5, y = var_509)[name = tensor("op_510")]; - tensor clip_7 = clip(alpha = var_22, beta = var_17, x = var_510)[name = tensor("clip_7")]; - tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; - tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_51)[name = tensor("x_11")]; - tensor input_53_perm_0 = const()[name = tensor("input_53_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_21_pad_type_0 = const()[name = tensor("inputs_21_pad_type_0"), val = tensor("valid")]; - tensor inputs_21_strides_0 = const()[name = tensor("inputs_21_strides_0"), val = tensor([1])]; - tensor inputs_21_pad_0 = const()[name = tensor("inputs_21_pad_0"), val = tensor([0, 0])]; - tensor inputs_21_dilations_0 = const()[name = tensor("inputs_21_dilations_0"), val = tensor([1])]; - tensor inputs_21_groups_0 = const()[name = tensor("inputs_21_groups_0"), val = tensor(1)]; - tensor input_53 = transpose(perm = input_53_perm_0, x = x_11)[name = tensor("transpose_76")]; - tensor inputs_21 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_21_dilations_0, groups = inputs_21_groups_0, pad = inputs_21_pad_0, pad_type = inputs_21_pad_type_0, strides = inputs_21_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_53)[name = tensor("inputs_21")]; - tensor var_531_split_sizes_0 = const()[name = tensor("op_531_split_sizes_0"), val = tensor([256, 256])]; - tensor var_531_axis_0 = const()[name = tensor("op_531_axis_0"), val = tensor(1)]; - tensor var_531_0, tensor var_531_1 = split(axis = var_531_axis_0, split_sizes = var_531_split_sizes_0, x = inputs_21)[name = tensor("op_531")]; - tensor var_533 = sigmoid(x = var_531_1)[name = tensor("op_533")]; - tensor inputs_23 = mul(x = var_531_0, y = var_533)[name = tensor("inputs_23")]; - tensor outputs_aug_7_pad_type_0 = const()[name = tensor("outputs_aug_7_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_7_pad_0 = const()[name = tensor("outputs_aug_7_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_7_groups_0 = const()[name = tensor("outputs_aug_7_groups_0"), val = tensor(256)]; - tensor outputs_aug_7_strides_0 = const()[name = tensor("outputs_aug_7_strides_0"), val = tensor([1])]; - tensor outputs_aug_7_dilations_0 = const()[name = tensor("outputs_aug_7_dilations_0"), val = tensor([1])]; - tensor outputs_aug_7 = conv(dilations = outputs_aug_7_dilations_0, groups = outputs_aug_7_groups_0, pad = outputs_aug_7_pad_0, pad_type = outputs_aug_7_pad_type_0, strides = outputs_aug_7_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_23)[name = tensor("outputs_aug_7")]; - tensor input_55_begin_0 = const()[name = tensor("input_55_begin_0"), val = tensor([0, 0, 0])]; - tensor input_55_end_0 = const()[name = tensor("input_55_end_0"), val = tensor([1, 256, 16])]; - tensor input_55_end_mask_0 = const()[name = tensor("input_55_end_mask_0"), val = tensor([true, true, false])]; - tensor input_55 = slice_by_index(begin = input_55_begin_0, end = input_55_end_0, end_mask = input_55_end_mask_0, x = outputs_aug_7)[name = tensor("input_55")]; - tensor inputs_25 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_55)[name = tensor("inputs_25")]; - tensor input_57 = silu(x = inputs_25)[name = tensor("input_57")]; - tensor input_59_pad_type_0 = const()[name = tensor("input_59_pad_type_0"), val = tensor("valid")]; - tensor input_59_strides_0 = const()[name = tensor("input_59_strides_0"), val = tensor([1])]; - tensor input_59_pad_0 = const()[name = tensor("input_59_pad_0"), val = tensor([0, 0])]; - tensor input_59_dilations_0 = const()[name = tensor("input_59_dilations_0"), val = tensor([1])]; - tensor input_59_groups_0 = const()[name = tensor("input_59_groups_0"), val = tensor(1)]; - tensor input_59 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_59_dilations_0, groups = input_59_groups_0, pad = input_59_pad_0, pad_type = input_59_pad_type_0, strides = input_59_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_57)[name = tensor("input_59")]; - tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; - tensor var_564_begin_0 = const()[name = tensor("op_564_begin_0"), val = tensor([0, -1, 0])]; - tensor var_564_end_0 = const()[name = tensor("op_564_end_0"), val = tensor([1, 16, 256])]; - tensor var_564_end_mask_0 = const()[name = tensor("op_564_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_59)[name = tensor("transpose_75")]; - tensor var_564 = slice_by_index(begin = var_564_begin_0, end = var_564_end_0, end_mask = var_564_end_mask_0, x = conv_out_7)[name = tensor("op_564")]; - tensor var_567_begin_0 = const()[name = tensor("op_567_begin_0"), val = tensor([0, 4, 0])]; - tensor var_567_end_0 = const()[name = tensor("op_567_end_0"), val = tensor([1, 1, 256])]; - tensor var_567_end_mask_0 = const()[name = tensor("op_567_end_mask_0"), val = tensor([true, true, true])]; - tensor var_567 = slice_by_index(begin = var_567_begin_0, end = var_567_end_0, end_mask = var_567_end_mask_0, x = x_3)[name = tensor("op_567")]; - tensor var_570_begin_0 = const()[name = tensor("op_570_begin_0"), val = tensor([0, 1, 0])]; - tensor var_570_end_0 = const()[name = tensor("op_570_end_0"), val = tensor([1, 16, 256])]; - tensor var_570_end_mask_0 = const()[name = tensor("op_570_end_mask_0"), val = tensor([true, true, true])]; - tensor var_570 = slice_by_index(begin = var_570_begin_0, end = var_570_end_0, end_mask = var_570_end_mask_0, x = input_51)[name = tensor("op_570")]; - tensor rolled_9_interleave_0 = const()[name = tensor("rolled_9_interleave_0"), val = tensor(false)]; - tensor rolled_9 = concat(axis = var_34, interleave = rolled_9_interleave_0, values = (var_570, var_567))[name = tensor("rolled_9")]; - tensor clip_8 = clip(alpha = var_22, beta = var_21, x = clip_7)[name = tensor("clip_8")]; - tensor var_575 = equal(x = encoder__conv_positions, y = clip_8)[name = tensor("op_575")]; - tensor var_576_dtype_0 = const()[name = tensor("op_576_dtype_0"), val = tensor("fp32")]; - tensor var_577 = const()[name = tensor("op_577"), val = tensor([1, 16, 1])]; - tensor var_576 = cast(dtype = var_576_dtype_0, x = var_575)[name = tensor("cast_183")]; - tensor fill_mask_9 = reshape(shape = var_577, x = var_576)[name = tensor("fill_mask_9")]; - tensor x_expanded_9_reps_0 = const()[name = tensor("x_expanded_9_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_9 = tile(reps = x_expanded_9_reps_0, x = var_567)[name = tensor("x_expanded_9")]; - tensor var_581 = sub(x = var_18, y = fill_mask_9)[name = tensor("op_581")]; - tensor var_582 = mul(x = input_51, y = var_581)[name = tensor("op_582")]; - tensor var_583 = mul(x = x_expanded_9, y = fill_mask_9)[name = tensor("op_583")]; - tensor filled_window_9 = add(x = var_582, y = var_583)[name = tensor("filled_window_9")]; - tensor var_585 = greater_equal(x = clip_7, y = var_17)[name = tensor("op_585")]; - tensor var_586_dtype_0 = const()[name = tensor("op_586_dtype_0"), val = tensor("fp32")]; - tensor var_587 = const()[name = tensor("op_587"), val = tensor([1, 1, 1])]; - tensor var_586 = cast(dtype = var_586_dtype_0, x = var_585)[name = tensor("cast_182")]; - tensor use_roll_9 = reshape(shape = var_587, x = var_586)[name = tensor("use_roll_9")]; - tensor var_589 = sub(x = var_18, y = use_roll_9)[name = tensor("op_589")]; - tensor var_590 = mul(x = filled_window_9, y = var_589)[name = tensor("op_590")]; - tensor var_591 = mul(x = rolled_9, y = use_roll_9)[name = tensor("op_591")]; - tensor input_61 = add(x = var_590, y = var_591)[name = tensor("input_61")]; - tensor var_593 = const()[name = tensor("op_593"), val = tensor(0x1p+0)]; - tensor var_594 = add(x = clip_7, y = var_593)[name = tensor("op_594")]; - tensor clip_9 = clip(alpha = var_22, beta = var_17, x = var_594)[name = tensor("clip_9")]; - tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; - tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_conv_module_0_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_0_weight, x = input_61)[name = tensor("x_13")]; - tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_27_pad_type_0 = const()[name = tensor("inputs_27_pad_type_0"), val = tensor("valid")]; - tensor inputs_27_strides_0 = const()[name = tensor("inputs_27_strides_0"), val = tensor([1])]; - tensor inputs_27_pad_0 = const()[name = tensor("inputs_27_pad_0"), val = tensor([0, 0])]; - tensor inputs_27_dilations_0 = const()[name = tensor("inputs_27_dilations_0"), val = tensor([1])]; - tensor inputs_27_groups_0 = const()[name = tensor("inputs_27_groups_0"), val = tensor(1)]; - tensor input_63 = transpose(perm = input_63_perm_0, x = x_13)[name = tensor("transpose_74")]; - tensor inputs_27 = conv(bias = encoder_conv_module_0_sequential_2_conv_bias, dilations = inputs_27_dilations_0, groups = inputs_27_groups_0, pad = inputs_27_pad_0, pad_type = inputs_27_pad_type_0, strides = inputs_27_strides_0, weight = encoder_conv_module_0_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_27")]; - tensor var_615_split_sizes_0 = const()[name = tensor("op_615_split_sizes_0"), val = tensor([256, 256])]; - tensor var_615_axis_0 = const()[name = tensor("op_615_axis_0"), val = tensor(1)]; - tensor var_615_0, tensor var_615_1 = split(axis = var_615_axis_0, split_sizes = var_615_split_sizes_0, x = inputs_27)[name = tensor("op_615")]; - tensor var_617 = sigmoid(x = var_615_1)[name = tensor("op_617")]; - tensor inputs_29 = mul(x = var_615_0, y = var_617)[name = tensor("inputs_29")]; - tensor outputs_aug_9_pad_type_0 = const()[name = tensor("outputs_aug_9_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_9_pad_0 = const()[name = tensor("outputs_aug_9_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_9_groups_0 = const()[name = tensor("outputs_aug_9_groups_0"), val = tensor(256)]; - tensor outputs_aug_9_strides_0 = const()[name = tensor("outputs_aug_9_strides_0"), val = tensor([1])]; - tensor outputs_aug_9_dilations_0 = const()[name = tensor("outputs_aug_9_dilations_0"), val = tensor([1])]; - tensor outputs_aug_9 = conv(dilations = outputs_aug_9_dilations_0, groups = outputs_aug_9_groups_0, pad = outputs_aug_9_pad_0, pad_type = outputs_aug_9_pad_type_0, strides = outputs_aug_9_strides_0, weight = encoder_conv_module_0_sequential_4_conv_weight, x = inputs_29)[name = tensor("outputs_aug_9")]; - tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; - tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([1, 256, 16])]; - tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; - tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_9)[name = tensor("input_65")]; - tensor inputs_31 = batch_norm(beta = encoder_conv_module_0_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_0_sequential_5_weight, mean = encoder_conv_module_0_sequential_5_running_mean, variance = encoder_conv_module_0_sequential_5_running_var, x = input_65)[name = tensor("inputs_31")]; - tensor input_67 = silu(x = inputs_31)[name = tensor("input_67")]; - tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; - tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; - tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; - tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; - tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; - tensor input_69 = conv(bias = encoder_conv_module_0_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_0_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; - tensor conv_out_9_perm_0 = const()[name = tensor("conv_out_9_perm_0"), val = tensor([0, 2, 1])]; - tensor var_648_begin_0 = const()[name = tensor("op_648_begin_0"), val = tensor([0, -1, 0])]; - tensor var_648_end_0 = const()[name = tensor("op_648_end_0"), val = tensor([1, 16, 256])]; - tensor var_648_end_mask_0 = const()[name = tensor("op_648_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_9 = transpose(perm = conv_out_9_perm_0, x = input_69)[name = tensor("transpose_73")]; - tensor var_648 = slice_by_index(begin = var_648_begin_0, end = var_648_end_0, end_mask = var_648_end_mask_0, x = conv_out_9)[name = tensor("op_648")]; - tensor var_651_interleave_0 = const()[name = tensor("op_651_interleave_0"), val = tensor(false)]; - tensor var_651 = concat(axis = var_34, interleave = var_651_interleave_0, values = (var_312, var_396, var_480, var_564, var_648))[name = tensor("op_651")]; - tensor input_71 = add(x = x_3, y = var_651)[name = tensor("input_71")]; - tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; - tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_0_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_0_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; - tensor inputs_33 = linear(bias = encoder_ffn2_0_module_sequential_1_linear_bias, weight = encoder_ffn2_0_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_8")]; - tensor input_75 = silu(x = inputs_33)[name = tensor("input_75")]; - tensor input_79 = linear(bias = encoder_ffn2_0_module_sequential_4_linear_bias, weight = encoder_ffn2_0_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_9")]; - tensor var_674 = const()[name = tensor("op_674"), val = tensor(0x1p-1)]; - tensor var_675 = mul(x = input_79, y = var_674)[name = tensor("op_675")]; - tensor input_81 = add(x = var_675, y = input_71)[name = tensor("input_81")]; - tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; - tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_0_bias, epsilon = var_36, gamma = encoder_layer_norm_0_weight, x = input_81)[name = tensor("input_83")]; - tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; - tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_1_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_1_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; - tensor inputs_35 = linear(bias = encoder_ffn1_1_module_sequential_1_linear_bias, weight = encoder_ffn1_1_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_10")]; - tensor input_87 = silu(x = inputs_35)[name = tensor("input_87")]; - tensor input_91 = linear(bias = encoder_ffn1_1_module_sequential_4_linear_bias, weight = encoder_ffn1_1_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_11")]; - tensor var_704 = const()[name = tensor("op_704"), val = tensor(0x1p-1)]; - tensor var_705 = mul(x = input_91, y = var_704)[name = tensor("op_705")]; - tensor input_93 = add(x = var_705, y = input_83)[name = tensor("input_93")]; - tensor x_15_axes_0 = const()[name = tensor("x_15_axes_0"), val = tensor([-1])]; - tensor x_15 = layer_norm(axes = x_15_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_36, gamma = encoder_ret_lns_1_weight, x = input_93)[name = tensor("x_15")]; + tensor x_7 = layer_norm(axes = x_7_axes_0, beta = encoder_ret_lns_1_bias, epsilon = var_36, gamma = encoder_ret_lns_1_weight, x = input_53)[name = tensor("x_7")]; tensor prev_kv_3_begin_0 = const()[name = tensor("prev_kv_3_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_3_end_0 = const()[name = tensor("prev_kv_3_end_0"), val = tensor([2, 1, 4, 64, 64])]; tensor prev_kv_3_end_mask_0 = const()[name = tensor("prev_kv_3_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -709,462 +556,308 @@ program(1.0) tensor prev_scale_3_end_mask_0 = const()[name = tensor("prev_scale_3_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_3_squeeze_mask_0 = const()[name = tensor("prev_scale_3_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_3 = slice_by_index(begin = prev_scale_3_begin_0, end = prev_scale_3_end_0, end_mask = prev_scale_3_end_mask_0, squeeze_mask = prev_scale_3_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_3")]; - tensor var_719 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_15)[name = tensor("linear_12")]; - tensor var_720 = const()[name = tensor("op_720"), val = tensor([1, 5, 4, 64])]; - tensor var_721 = reshape(shape = var_720, x = var_719)[name = tensor("op_721")]; + tensor var_504 = linear(bias = encoder_q_proj_1_bias, weight = encoder_q_proj_1_weight, x = x_7)[name = tensor("linear_12")]; + tensor var_505 = const()[name = tensor("op_505"), val = tensor([1, 5, 4, 64])]; + tensor var_506 = reshape(shape = var_505, x = var_504)[name = tensor("op_506")]; tensor q_3_perm_0 = const()[name = tensor("q_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_725 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_15)[name = tensor("linear_13")]; - tensor var_726 = const()[name = tensor("op_726"), val = tensor(0x1p-3)]; - tensor var_727 = mul(x = var_725, y = var_726)[name = tensor("op_727")]; - tensor var_728 = const()[name = tensor("op_728"), val = tensor([1, 5, 4, 64])]; - tensor var_729 = reshape(shape = var_728, x = var_727)[name = tensor("op_729")]; + tensor var_510 = linear(bias = encoder_k_proj_1_bias, weight = encoder_k_proj_1_weight, x = x_7)[name = tensor("linear_13")]; + tensor var_511 = const()[name = tensor("op_511"), val = tensor(0x1p-3)]; + tensor var_512 = mul(x = var_510, y = var_511)[name = tensor("op_512")]; + tensor var_513 = const()[name = tensor("op_513"), val = tensor([1, 5, 4, 64])]; + tensor var_514 = reshape(shape = var_513, x = var_512)[name = tensor("op_514")]; tensor k_3_perm_0 = const()[name = tensor("k_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_733 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_15)[name = tensor("linear_14")]; - tensor var_734 = const()[name = tensor("op_734"), val = tensor([1, 5, 4, 64])]; - tensor var_735 = reshape(shape = var_734, x = var_733)[name = tensor("op_735")]; + tensor var_518 = linear(bias = encoder_v_proj_1_bias, weight = encoder_v_proj_1_weight, x = x_7)[name = tensor("linear_14")]; + tensor var_519 = const()[name = tensor("op_519"), val = tensor([1, 5, 4, 64])]; + tensor var_520 = reshape(shape = var_519, x = var_518)[name = tensor("op_520")]; tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_97 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_15)[name = tensor("linear_15")]; + tensor input_57 = linear(bias = encoder_g_proj_1_bias, weight = encoder_g_proj_1_weight, x = x_7)[name = tensor("linear_15")]; tensor sqrt_s0_3 = sqrt(x = prev_scale_3)[name = tensor("sqrt_s0_3")]; tensor s_t_3 = add(x = prev_scale_3, y = encoder__t_index)[name = tensor("s_t_3")]; tensor sqrt_s_t_3 = sqrt(x = s_t_3)[name = tensor("sqrt_s_t_3")]; tensor qk_3_transpose_x_1 = const()[name = tensor("qk_3_transpose_x_1"), val = tensor(false)]; tensor qk_3_transpose_y_1 = const()[name = tensor("qk_3_transpose_y_1"), val = tensor(true)]; - tensor k_3 = transpose(perm = k_3_perm_0, x = var_729)[name = tensor("transpose_71")]; - tensor q_3 = transpose(perm = q_3_perm_0, x = var_721)[name = tensor("transpose_72")]; + tensor k_3 = transpose(perm = k_3_perm_0, x = var_514)[name = tensor("transpose_50")]; + tensor q_3 = transpose(perm = q_3_perm_0, x = var_506)[name = tensor("transpose_51")]; tensor qk_3 = matmul(transpose_x = qk_3_transpose_x_1, transpose_y = qk_3_transpose_y_1, x = q_3, y = k_3)[name = tensor("qk_3")]; - tensor var_745 = const()[name = tensor("op_745"), val = tensor([5, 1])]; - tensor var_746 = reshape(shape = var_745, x = sqrt_s_t_3)[name = tensor("op_746")]; - tensor M_3 = real_div(x = encoder__causal_mask, y = var_746)[name = tensor("M_3")]; - tensor var_748 = mul(x = qk_3, y = M_3)[name = tensor("op_748")]; + tensor var_530 = const()[name = tensor("op_530"), val = tensor([5, 1])]; + tensor var_531 = reshape(shape = var_530, x = sqrt_s_t_3)[name = tensor("op_531")]; + tensor M_3 = real_div(x = encoder__causal_mask, y = var_531)[name = tensor("M_3")]; + tensor var_533 = mul(x = qk_3, y = M_3)[name = tensor("op_533")]; tensor inner_3_transpose_x_0 = const()[name = tensor("inner_3_transpose_x_0"), val = tensor(false)]; tensor inner_3_transpose_y_0 = const()[name = tensor("inner_3_transpose_y_0"), val = tensor(false)]; - tensor v_3 = transpose(perm = v_3_perm_0, x = var_735)[name = tensor("transpose_70")]; - tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_748, y = v_3)[name = tensor("inner_3")]; - tensor var_750_transpose_x_0 = const()[name = tensor("op_750_transpose_x_0"), val = tensor(false)]; - tensor var_750_transpose_y_0 = const()[name = tensor("op_750_transpose_y_0"), val = tensor(false)]; - tensor var_750 = matmul(transpose_x = var_750_transpose_x_0, transpose_y = var_750_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_750")]; - tensor var_751 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_751")]; - tensor var_752 = const()[name = tensor("op_752"), val = tensor([1, 1, 5, 1])]; - tensor var_753 = reshape(shape = var_752, x = var_751)[name = tensor("op_753")]; - tensor cross_3 = mul(x = var_750, y = var_753)[name = tensor("cross_3")]; + tensor v_3 = transpose(perm = v_3_perm_0, x = var_520)[name = tensor("transpose_49")]; + tensor inner_3 = matmul(transpose_x = inner_3_transpose_x_0, transpose_y = inner_3_transpose_y_0, x = var_533, y = v_3)[name = tensor("inner_3")]; + tensor var_535_transpose_x_0 = const()[name = tensor("op_535_transpose_x_0"), val = tensor(false)]; + tensor var_535_transpose_y_0 = const()[name = tensor("op_535_transpose_y_0"), val = tensor(false)]; + tensor var_535 = matmul(transpose_x = var_535_transpose_x_0, transpose_y = var_535_transpose_y_0, x = q_3, y = prev_kv_3)[name = tensor("op_535")]; + tensor var_536 = real_div(x = sqrt_s0_3, y = sqrt_s_t_3)[name = tensor("op_536")]; + tensor var_537 = const()[name = tensor("op_537"), val = tensor([1, 1, 5, 1])]; + tensor var_538 = reshape(shape = var_537, x = var_536)[name = tensor("op_538")]; + tensor cross_3 = mul(x = var_535, y = var_538)[name = tensor("cross_3")]; tensor out_7 = add(x = inner_3, y = cross_3)[name = tensor("out_7")]; - tensor var_756 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_756")]; - tensor var_758_transpose_x_1 = const()[name = tensor("op_758_transpose_x_1"), val = tensor(true)]; - tensor var_758_transpose_y_1 = const()[name = tensor("op_758_transpose_y_1"), val = tensor(false)]; - tensor var_758 = matmul(transpose_x = var_758_transpose_x_1, transpose_y = var_758_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_758")]; - tensor new_kv_unnorm_3 = add(x = var_756, y = var_758)[name = tensor("new_kv_unnorm_3")]; - tensor var_760 = const()[name = tensor("op_760"), val = tensor(0x1.4p+2)]; - tensor new_scale_3 = add(x = prev_scale_3, y = var_760)[name = tensor("new_scale_3")]; - tensor var_762 = sqrt(x = new_scale_3)[name = tensor("op_762")]; - tensor var_763 = real_div(x = new_kv_unnorm_3, y = var_762)[name = tensor("op_763")]; - tensor var_764_perm_0 = const()[name = tensor("op_764_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_541 = mul(x = prev_kv_3, y = sqrt_s0_3)[name = tensor("op_541")]; + tensor var_543_transpose_x_1 = const()[name = tensor("op_543_transpose_x_1"), val = tensor(true)]; + tensor var_543_transpose_y_1 = const()[name = tensor("op_543_transpose_y_1"), val = tensor(false)]; + tensor var_543 = matmul(transpose_x = var_543_transpose_x_1, transpose_y = var_543_transpose_y_1, x = k_3, y = v_3)[name = tensor("op_543")]; + tensor new_kv_unnorm_3 = add(x = var_541, y = var_543)[name = tensor("new_kv_unnorm_3")]; + tensor var_545 = const()[name = tensor("op_545"), val = tensor(0x1.4p+2)]; + tensor new_scale_3 = add(x = prev_scale_3, y = var_545)[name = tensor("new_scale_3")]; + tensor var_547 = sqrt(x = new_scale_3)[name = tensor("op_547")]; + tensor var_548 = real_div(x = new_kv_unnorm_3, y = var_547)[name = tensor("op_548")]; + tensor var_549_perm_0 = const()[name = tensor("op_549_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_9_axes_0 = const()[name = tensor("out_9_axes_0"), val = tensor([-1])]; - tensor var_764 = transpose(perm = var_764_perm_0, x = out_7)[name = tensor("transpose_69")]; - tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_25, x = var_764)[name = tensor("out_9")]; - tensor var_768 = const()[name = tensor("op_768"), val = tensor([1, 5, 256])]; - tensor out_11 = reshape(shape = var_768, x = out_9)[name = tensor("out_11")]; - tensor var_770 = silu(x = input_97)[name = tensor("op_770")]; - tensor input_99 = mul(x = var_770, y = out_11)[name = tensor("input_99")]; - tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_99)[name = tensor("linear_16")]; - tensor x_17 = add(x = input_93, y = ret_out_3)[name = tensor("x_17")]; - tensor window_3_begin_0 = const()[name = tensor("window_3_begin_0"), val = tensor([1, 0, 0, 0])]; - tensor window_3_end_0 = const()[name = tensor("window_3_end_0"), val = tensor([2, 1, 16, 256])]; - tensor window_3_end_mask_0 = const()[name = tensor("window_3_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_3_squeeze_mask_0 = const()[name = tensor("window_3_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_3 = slice_by_index(begin = window_3_begin_0, end = window_3_end_0, end_mask = window_3_end_mask_0, squeeze_mask = window_3_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_3")]; + tensor var_549 = transpose(perm = var_549_perm_0, x = out_7)[name = tensor("transpose_48")]; + tensor out_9 = layer_norm(axes = out_9_axes_0, epsilon = var_25, x = var_549)[name = tensor("out_9")]; + tensor var_553 = const()[name = tensor("op_553"), val = tensor([1, 5, 256])]; + tensor out_11 = reshape(shape = var_553, x = out_9)[name = tensor("out_11")]; + tensor var_555 = silu(x = input_57)[name = tensor("op_555")]; + tensor input_59 = mul(x = var_555, y = out_11)[name = tensor("input_59")]; + tensor ret_out_3 = linear(bias = encoder_out_proj_1_bias, weight = encoder_out_proj_1_weight, x = input_59)[name = tensor("linear_16")]; + tensor x_9 = add(x = input_53, y = ret_out_3)[name = tensor("x_9")]; + tensor window_13_begin_0 = const()[name = tensor("window_13_begin_0"), val = tensor([1, 0, 0, 0])]; + tensor window_13_end_0 = const()[name = tensor("window_13_end_0"), val = tensor([2, 1, 16, 256])]; + tensor window_13_end_mask_0 = const()[name = tensor("window_13_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_13_squeeze_mask_0 = const()[name = tensor("window_13_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_13 = slice_by_index(begin = window_13_begin_0, end = window_13_end_0, end_mask = window_13_end_mask_0, squeeze_mask = window_13_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_13")]; tensor filled_11_begin_0 = const()[name = tensor("filled_11_begin_0"), val = tensor([1, 0])]; tensor filled_11_end_0 = const()[name = tensor("filled_11_end_0"), val = tensor([2, 1])]; tensor filled_11_end_mask_0 = const()[name = tensor("filled_11_end_mask_0"), val = tensor([false, true])]; tensor filled_11_squeeze_mask_0 = const()[name = tensor("filled_11_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_11 = slice_by_index(begin = filled_11_begin_0, end = filled_11_end_0, end_mask = filled_11_end_mask_0, squeeze_mask = filled_11_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_11")]; - tensor var_779_begin_0 = const()[name = tensor("op_779_begin_0"), val = tensor([0, 0, 0])]; - tensor var_779_end_0 = const()[name = tensor("op_779_end_0"), val = tensor([1, 1, 256])]; - tensor var_779_end_mask_0 = const()[name = tensor("op_779_end_mask_0"), val = tensor([true, false, true])]; - tensor var_779 = slice_by_index(begin = var_779_begin_0, end = var_779_end_0, end_mask = var_779_end_mask_0, x = x_17)[name = tensor("op_779")]; - tensor var_782_begin_0 = const()[name = tensor("op_782_begin_0"), val = tensor([0, 1, 0])]; - tensor var_782_end_0 = const()[name = tensor("op_782_end_0"), val = tensor([1, 16, 256])]; - tensor var_782_end_mask_0 = const()[name = tensor("op_782_end_mask_0"), val = tensor([true, true, true])]; - tensor var_782 = slice_by_index(begin = var_782_begin_0, end = var_782_end_0, end_mask = var_782_end_mask_0, x = window_3)[name = tensor("op_782")]; + tensor var_564_begin_0 = const()[name = tensor("op_564_begin_0"), val = tensor([0, 0, 0])]; + tensor var_564_end_0 = const()[name = tensor("op_564_end_0"), val = tensor([1, 1, 256])]; + tensor var_564_end_mask_0 = const()[name = tensor("op_564_end_mask_0"), val = tensor([true, false, true])]; + tensor var_564 = slice_by_index(begin = var_564_begin_0, end = var_564_end_0, end_mask = var_564_end_mask_0, x = x_9)[name = tensor("op_564")]; + tensor var_567_begin_0 = const()[name = tensor("op_567_begin_0"), val = tensor([0, 1, 0])]; + tensor var_567_end_0 = const()[name = tensor("op_567_end_0"), val = tensor([1, 16, 256])]; + tensor var_567_end_mask_0 = const()[name = tensor("op_567_end_mask_0"), val = tensor([true, true, true])]; + tensor var_567 = slice_by_index(begin = var_567_begin_0, end = var_567_end_0, end_mask = var_567_end_mask_0, x = window_13)[name = tensor("op_567")]; tensor rolled_11_interleave_0 = const()[name = tensor("rolled_11_interleave_0"), val = tensor(false)]; - tensor rolled_11 = concat(axis = var_34, interleave = rolled_11_interleave_0, values = (var_782, var_779))[name = tensor("rolled_11")]; + tensor rolled_11 = concat(axis = var_34, interleave = rolled_11_interleave_0, values = (var_567, var_564))[name = tensor("rolled_11")]; tensor clip_10 = clip(alpha = var_22, beta = var_21, x = filled_11)[name = tensor("clip_10")]; - tensor var_787 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_787")]; - tensor var_788_dtype_0 = const()[name = tensor("op_788_dtype_0"), val = tensor("fp32")]; - tensor var_789 = const()[name = tensor("op_789"), val = tensor([1, 16, 1])]; - tensor var_788 = cast(dtype = var_788_dtype_0, x = var_787)[name = tensor("cast_181")]; - tensor fill_mask_11 = reshape(shape = var_789, x = var_788)[name = tensor("fill_mask_11")]; + tensor var_572 = equal(x = encoder__conv_positions, y = clip_10)[name = tensor("op_572")]; + tensor var_573_dtype_0 = const()[name = tensor("op_573_dtype_0"), val = tensor("fp32")]; + tensor var_574 = const()[name = tensor("op_574"), val = tensor([1, 16, 1])]; + tensor var_573 = cast(dtype = var_573_dtype_0, x = var_572)[name = tensor("cast_181")]; + tensor fill_mask_11 = reshape(shape = var_574, x = var_573)[name = tensor("fill_mask_11")]; tensor x_expanded_11_reps_0 = const()[name = tensor("x_expanded_11_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_779)[name = tensor("x_expanded_11")]; - tensor var_793 = sub(x = var_18, y = fill_mask_11)[name = tensor("op_793")]; - tensor var_794 = mul(x = window_3, y = var_793)[name = tensor("op_794")]; - tensor var_795 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_795")]; - tensor filled_window_11 = add(x = var_794, y = var_795)[name = tensor("filled_window_11")]; - tensor var_797 = greater_equal(x = filled_11, y = var_17)[name = tensor("op_797")]; - tensor var_798_dtype_0 = const()[name = tensor("op_798_dtype_0"), val = tensor("fp32")]; - tensor var_799 = const()[name = tensor("op_799"), val = tensor([1, 1, 1])]; - tensor var_798 = cast(dtype = var_798_dtype_0, x = var_797)[name = tensor("cast_180")]; - tensor use_roll_11 = reshape(shape = var_799, x = var_798)[name = tensor("use_roll_11")]; - tensor var_801 = sub(x = var_18, y = use_roll_11)[name = tensor("op_801")]; - tensor var_802 = mul(x = filled_window_11, y = var_801)[name = tensor("op_802")]; - tensor var_803 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_803")]; - tensor input_101 = add(x = var_802, y = var_803)[name = tensor("input_101")]; - tensor var_805 = const()[name = tensor("op_805"), val = tensor(0x1p+0)]; - tensor var_806 = add(x = filled_11, y = var_805)[name = tensor("op_806")]; - tensor clip_11 = clip(alpha = var_22, beta = var_17, x = var_806)[name = tensor("clip_11")]; - tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; - tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_101)[name = tensor("x_19")]; - tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_37_pad_type_0 = const()[name = tensor("inputs_37_pad_type_0"), val = tensor("valid")]; - tensor inputs_37_strides_0 = const()[name = tensor("inputs_37_strides_0"), val = tensor([1])]; - tensor inputs_37_pad_0 = const()[name = tensor("inputs_37_pad_0"), val = tensor([0, 0])]; - tensor inputs_37_dilations_0 = const()[name = tensor("inputs_37_dilations_0"), val = tensor([1])]; - tensor inputs_37_groups_0 = const()[name = tensor("inputs_37_groups_0"), val = tensor(1)]; - tensor input_103 = transpose(perm = input_103_perm_0, x = x_19)[name = tensor("transpose_68")]; - tensor inputs_37 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_37_dilations_0, groups = inputs_37_groups_0, pad = inputs_37_pad_0, pad_type = inputs_37_pad_type_0, strides = inputs_37_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_37")]; - tensor var_827_split_sizes_0 = const()[name = tensor("op_827_split_sizes_0"), val = tensor([256, 256])]; - tensor var_827_axis_0 = const()[name = tensor("op_827_axis_0"), val = tensor(1)]; - tensor var_827_0, tensor var_827_1 = split(axis = var_827_axis_0, split_sizes = var_827_split_sizes_0, x = inputs_37)[name = tensor("op_827")]; - tensor var_829 = sigmoid(x = var_827_1)[name = tensor("op_829")]; - tensor inputs_39 = mul(x = var_827_0, y = var_829)[name = tensor("inputs_39")]; - tensor outputs_aug_11_pad_type_0 = const()[name = tensor("outputs_aug_11_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_11_pad_0 = const()[name = tensor("outputs_aug_11_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_11_groups_0 = const()[name = tensor("outputs_aug_11_groups_0"), val = tensor(256)]; - tensor outputs_aug_11_strides_0 = const()[name = tensor("outputs_aug_11_strides_0"), val = tensor([1])]; - tensor outputs_aug_11_dilations_0 = const()[name = tensor("outputs_aug_11_dilations_0"), val = tensor([1])]; - tensor outputs_aug_11 = conv(dilations = outputs_aug_11_dilations_0, groups = outputs_aug_11_groups_0, pad = outputs_aug_11_pad_0, pad_type = outputs_aug_11_pad_type_0, strides = outputs_aug_11_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_39)[name = tensor("outputs_aug_11")]; - tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; - tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([1, 256, 16])]; - tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; - tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_11)[name = tensor("input_105")]; - tensor inputs_41 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_105)[name = tensor("inputs_41")]; - tensor input_107 = silu(x = inputs_41)[name = tensor("input_107")]; - tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; - tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; - tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; - tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; - tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; - tensor input_109 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; - tensor conv_out_11_perm_0 = const()[name = tensor("conv_out_11_perm_0"), val = tensor([0, 2, 1])]; - tensor var_860_begin_0 = const()[name = tensor("op_860_begin_0"), val = tensor([0, -1, 0])]; - tensor var_860_end_0 = const()[name = tensor("op_860_end_0"), val = tensor([1, 16, 256])]; - tensor var_860_end_mask_0 = const()[name = tensor("op_860_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_11 = transpose(perm = conv_out_11_perm_0, x = input_109)[name = tensor("transpose_67")]; - tensor var_860 = slice_by_index(begin = var_860_begin_0, end = var_860_end_0, end_mask = var_860_end_mask_0, x = conv_out_11)[name = tensor("op_860")]; - tensor var_863_begin_0 = const()[name = tensor("op_863_begin_0"), val = tensor([0, 1, 0])]; - tensor var_863_end_0 = const()[name = tensor("op_863_end_0"), val = tensor([1, 2, 256])]; - tensor var_863_end_mask_0 = const()[name = tensor("op_863_end_mask_0"), val = tensor([true, false, true])]; - tensor var_863 = slice_by_index(begin = var_863_begin_0, end = var_863_end_0, end_mask = var_863_end_mask_0, x = x_17)[name = tensor("op_863")]; - tensor var_866_begin_0 = const()[name = tensor("op_866_begin_0"), val = tensor([0, 1, 0])]; - tensor var_866_end_0 = const()[name = tensor("op_866_end_0"), val = tensor([1, 16, 256])]; - tensor var_866_end_mask_0 = const()[name = tensor("op_866_end_mask_0"), val = tensor([true, true, true])]; - tensor var_866 = slice_by_index(begin = var_866_begin_0, end = var_866_end_0, end_mask = var_866_end_mask_0, x = input_101)[name = tensor("op_866")]; + tensor x_expanded_11 = tile(reps = x_expanded_11_reps_0, x = var_564)[name = tensor("x_expanded_11")]; + tensor var_578 = sub(x = var_18, y = fill_mask_11)[name = tensor("op_578")]; + tensor var_579 = mul(x = window_13, y = var_578)[name = tensor("op_579")]; + tensor var_580 = mul(x = x_expanded_11, y = fill_mask_11)[name = tensor("op_580")]; + tensor filled_window_11 = add(x = var_579, y = var_580)[name = tensor("filled_window_11")]; + tensor var_582 = greater_equal(x = filled_11, y = var_17)[name = tensor("op_582")]; + tensor var_583_dtype_0 = const()[name = tensor("op_583_dtype_0"), val = tensor("fp32")]; + tensor var_584 = const()[name = tensor("op_584"), val = tensor([1, 1, 1])]; + tensor var_583 = cast(dtype = var_583_dtype_0, x = var_582)[name = tensor("cast_180")]; + tensor use_roll_11 = reshape(shape = var_584, x = var_583)[name = tensor("use_roll_11")]; + tensor var_586 = sub(x = var_18, y = use_roll_11)[name = tensor("op_586")]; + tensor var_587 = mul(x = filled_window_11, y = var_586)[name = tensor("op_587")]; + tensor var_588 = mul(x = rolled_11, y = use_roll_11)[name = tensor("op_588")]; + tensor window_15 = add(x = var_587, y = var_588)[name = tensor("window_15")]; + tensor var_590 = const()[name = tensor("op_590"), val = tensor(0x1p+0)]; + tensor var_591 = add(x = filled_11, y = var_590)[name = tensor("op_591")]; + tensor clip_11 = clip(alpha = var_22, beta = var_17, x = var_591)[name = tensor("clip_11")]; + tensor var_594_begin_0 = const()[name = tensor("op_594_begin_0"), val = tensor([0, 1, 0])]; + tensor var_594_end_0 = const()[name = tensor("op_594_end_0"), val = tensor([1, 2, 256])]; + tensor var_594_end_mask_0 = const()[name = tensor("op_594_end_mask_0"), val = tensor([true, false, true])]; + tensor var_594 = slice_by_index(begin = var_594_begin_0, end = var_594_end_0, end_mask = var_594_end_mask_0, x = x_9)[name = tensor("op_594")]; + tensor var_597_begin_0 = const()[name = tensor("op_597_begin_0"), val = tensor([0, 1, 0])]; + tensor var_597_end_0 = const()[name = tensor("op_597_end_0"), val = tensor([1, 16, 256])]; + tensor var_597_end_mask_0 = const()[name = tensor("op_597_end_mask_0"), val = tensor([true, true, true])]; + tensor var_597 = slice_by_index(begin = var_597_begin_0, end = var_597_end_0, end_mask = var_597_end_mask_0, x = window_15)[name = tensor("op_597")]; tensor rolled_13_interleave_0 = const()[name = tensor("rolled_13_interleave_0"), val = tensor(false)]; - tensor rolled_13 = concat(axis = var_34, interleave = rolled_13_interleave_0, values = (var_866, var_863))[name = tensor("rolled_13")]; + tensor rolled_13 = concat(axis = var_34, interleave = rolled_13_interleave_0, values = (var_597, var_594))[name = tensor("rolled_13")]; tensor clip_12 = clip(alpha = var_22, beta = var_21, x = clip_11)[name = tensor("clip_12")]; - tensor var_871 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_871")]; - tensor var_872_dtype_0 = const()[name = tensor("op_872_dtype_0"), val = tensor("fp32")]; - tensor var_873 = const()[name = tensor("op_873"), val = tensor([1, 16, 1])]; - tensor var_872 = cast(dtype = var_872_dtype_0, x = var_871)[name = tensor("cast_179")]; - tensor fill_mask_13 = reshape(shape = var_873, x = var_872)[name = tensor("fill_mask_13")]; + tensor var_602 = equal(x = encoder__conv_positions, y = clip_12)[name = tensor("op_602")]; + tensor var_603_dtype_0 = const()[name = tensor("op_603_dtype_0"), val = tensor("fp32")]; + tensor var_604 = const()[name = tensor("op_604"), val = tensor([1, 16, 1])]; + tensor var_603 = cast(dtype = var_603_dtype_0, x = var_602)[name = tensor("cast_179")]; + tensor fill_mask_13 = reshape(shape = var_604, x = var_603)[name = tensor("fill_mask_13")]; tensor x_expanded_13_reps_0 = const()[name = tensor("x_expanded_13_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_863)[name = tensor("x_expanded_13")]; - tensor var_877 = sub(x = var_18, y = fill_mask_13)[name = tensor("op_877")]; - tensor var_878 = mul(x = input_101, y = var_877)[name = tensor("op_878")]; - tensor var_879 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_879")]; - tensor filled_window_13 = add(x = var_878, y = var_879)[name = tensor("filled_window_13")]; - tensor var_881 = greater_equal(x = clip_11, y = var_17)[name = tensor("op_881")]; - tensor var_882_dtype_0 = const()[name = tensor("op_882_dtype_0"), val = tensor("fp32")]; - tensor var_883 = const()[name = tensor("op_883"), val = tensor([1, 1, 1])]; - tensor var_882 = cast(dtype = var_882_dtype_0, x = var_881)[name = tensor("cast_178")]; - tensor use_roll_13 = reshape(shape = var_883, x = var_882)[name = tensor("use_roll_13")]; - tensor var_885 = sub(x = var_18, y = use_roll_13)[name = tensor("op_885")]; - tensor var_886 = mul(x = filled_window_13, y = var_885)[name = tensor("op_886")]; - tensor var_887 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_887")]; - tensor input_111 = add(x = var_886, y = var_887)[name = tensor("input_111")]; - tensor var_889 = const()[name = tensor("op_889"), val = tensor(0x1p+0)]; - tensor var_890 = add(x = clip_11, y = var_889)[name = tensor("op_890")]; - tensor clip_13 = clip(alpha = var_22, beta = var_17, x = var_890)[name = tensor("clip_13")]; - tensor x_21_axes_0 = const()[name = tensor("x_21_axes_0"), val = tensor([-1])]; - tensor x_21 = layer_norm(axes = x_21_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_111)[name = tensor("x_21")]; - tensor input_113_perm_0 = const()[name = tensor("input_113_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_43_pad_type_0 = const()[name = tensor("inputs_43_pad_type_0"), val = tensor("valid")]; - tensor inputs_43_strides_0 = const()[name = tensor("inputs_43_strides_0"), val = tensor([1])]; - tensor inputs_43_pad_0 = const()[name = tensor("inputs_43_pad_0"), val = tensor([0, 0])]; - tensor inputs_43_dilations_0 = const()[name = tensor("inputs_43_dilations_0"), val = tensor([1])]; - tensor inputs_43_groups_0 = const()[name = tensor("inputs_43_groups_0"), val = tensor(1)]; - tensor input_113 = transpose(perm = input_113_perm_0, x = x_21)[name = tensor("transpose_66")]; - tensor inputs_43 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_43_dilations_0, groups = inputs_43_groups_0, pad = inputs_43_pad_0, pad_type = inputs_43_pad_type_0, strides = inputs_43_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_113)[name = tensor("inputs_43")]; - tensor var_911_split_sizes_0 = const()[name = tensor("op_911_split_sizes_0"), val = tensor([256, 256])]; - tensor var_911_axis_0 = const()[name = tensor("op_911_axis_0"), val = tensor(1)]; - tensor var_911_0, tensor var_911_1 = split(axis = var_911_axis_0, split_sizes = var_911_split_sizes_0, x = inputs_43)[name = tensor("op_911")]; - tensor var_913 = sigmoid(x = var_911_1)[name = tensor("op_913")]; - tensor inputs_45 = mul(x = var_911_0, y = var_913)[name = tensor("inputs_45")]; - tensor outputs_aug_13_pad_type_0 = const()[name = tensor("outputs_aug_13_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_13_pad_0 = const()[name = tensor("outputs_aug_13_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_13_groups_0 = const()[name = tensor("outputs_aug_13_groups_0"), val = tensor(256)]; - tensor outputs_aug_13_strides_0 = const()[name = tensor("outputs_aug_13_strides_0"), val = tensor([1])]; - tensor outputs_aug_13_dilations_0 = const()[name = tensor("outputs_aug_13_dilations_0"), val = tensor([1])]; - tensor outputs_aug_13 = conv(dilations = outputs_aug_13_dilations_0, groups = outputs_aug_13_groups_0, pad = outputs_aug_13_pad_0, pad_type = outputs_aug_13_pad_type_0, strides = outputs_aug_13_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_45)[name = tensor("outputs_aug_13")]; - tensor input_115_begin_0 = const()[name = tensor("input_115_begin_0"), val = tensor([0, 0, 0])]; - tensor input_115_end_0 = const()[name = tensor("input_115_end_0"), val = tensor([1, 256, 16])]; - tensor input_115_end_mask_0 = const()[name = tensor("input_115_end_mask_0"), val = tensor([true, true, false])]; - tensor input_115 = slice_by_index(begin = input_115_begin_0, end = input_115_end_0, end_mask = input_115_end_mask_0, x = outputs_aug_13)[name = tensor("input_115")]; - tensor inputs_47 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_115)[name = tensor("inputs_47")]; - tensor input_117 = silu(x = inputs_47)[name = tensor("input_117")]; - tensor input_119_pad_type_0 = const()[name = tensor("input_119_pad_type_0"), val = tensor("valid")]; - tensor input_119_strides_0 = const()[name = tensor("input_119_strides_0"), val = tensor([1])]; - tensor input_119_pad_0 = const()[name = tensor("input_119_pad_0"), val = tensor([0, 0])]; - tensor input_119_dilations_0 = const()[name = tensor("input_119_dilations_0"), val = tensor([1])]; - tensor input_119_groups_0 = const()[name = tensor("input_119_groups_0"), val = tensor(1)]; - tensor input_119 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_119_dilations_0, groups = input_119_groups_0, pad = input_119_pad_0, pad_type = input_119_pad_type_0, strides = input_119_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_117)[name = tensor("input_119")]; - tensor conv_out_13_perm_0 = const()[name = tensor("conv_out_13_perm_0"), val = tensor([0, 2, 1])]; - tensor var_944_begin_0 = const()[name = tensor("op_944_begin_0"), val = tensor([0, -1, 0])]; - tensor var_944_end_0 = const()[name = tensor("op_944_end_0"), val = tensor([1, 16, 256])]; - tensor var_944_end_mask_0 = const()[name = tensor("op_944_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_13 = transpose(perm = conv_out_13_perm_0, x = input_119)[name = tensor("transpose_65")]; - tensor var_944 = slice_by_index(begin = var_944_begin_0, end = var_944_end_0, end_mask = var_944_end_mask_0, x = conv_out_13)[name = tensor("op_944")]; - tensor var_947_begin_0 = const()[name = tensor("op_947_begin_0"), val = tensor([0, 2, 0])]; - tensor var_947_end_0 = const()[name = tensor("op_947_end_0"), val = tensor([1, 3, 256])]; - tensor var_947_end_mask_0 = const()[name = tensor("op_947_end_mask_0"), val = tensor([true, false, true])]; - tensor var_947 = slice_by_index(begin = var_947_begin_0, end = var_947_end_0, end_mask = var_947_end_mask_0, x = x_17)[name = tensor("op_947")]; - tensor var_950_begin_0 = const()[name = tensor("op_950_begin_0"), val = tensor([0, 1, 0])]; - tensor var_950_end_0 = const()[name = tensor("op_950_end_0"), val = tensor([1, 16, 256])]; - tensor var_950_end_mask_0 = const()[name = tensor("op_950_end_mask_0"), val = tensor([true, true, true])]; - tensor var_950 = slice_by_index(begin = var_950_begin_0, end = var_950_end_0, end_mask = var_950_end_mask_0, x = input_111)[name = tensor("op_950")]; + tensor x_expanded_13 = tile(reps = x_expanded_13_reps_0, x = var_594)[name = tensor("x_expanded_13")]; + tensor var_608 = sub(x = var_18, y = fill_mask_13)[name = tensor("op_608")]; + tensor var_609 = mul(x = window_15, y = var_608)[name = tensor("op_609")]; + tensor var_610 = mul(x = x_expanded_13, y = fill_mask_13)[name = tensor("op_610")]; + tensor filled_window_13 = add(x = var_609, y = var_610)[name = tensor("filled_window_13")]; + tensor var_612 = greater_equal(x = clip_11, y = var_17)[name = tensor("op_612")]; + tensor var_613_dtype_0 = const()[name = tensor("op_613_dtype_0"), val = tensor("fp32")]; + tensor var_614 = const()[name = tensor("op_614"), val = tensor([1, 1, 1])]; + tensor var_613 = cast(dtype = var_613_dtype_0, x = var_612)[name = tensor("cast_178")]; + tensor use_roll_13 = reshape(shape = var_614, x = var_613)[name = tensor("use_roll_13")]; + tensor var_616 = sub(x = var_18, y = use_roll_13)[name = tensor("op_616")]; + tensor var_617 = mul(x = filled_window_13, y = var_616)[name = tensor("op_617")]; + tensor var_618 = mul(x = rolled_13, y = use_roll_13)[name = tensor("op_618")]; + tensor window_17 = add(x = var_617, y = var_618)[name = tensor("window_17")]; + tensor var_620 = const()[name = tensor("op_620"), val = tensor(0x1p+0)]; + tensor var_621 = add(x = clip_11, y = var_620)[name = tensor("op_621")]; + tensor clip_13 = clip(alpha = var_22, beta = var_17, x = var_621)[name = tensor("clip_13")]; + tensor var_624_begin_0 = const()[name = tensor("op_624_begin_0"), val = tensor([0, 2, 0])]; + tensor var_624_end_0 = const()[name = tensor("op_624_end_0"), val = tensor([1, 3, 256])]; + tensor var_624_end_mask_0 = const()[name = tensor("op_624_end_mask_0"), val = tensor([true, false, true])]; + tensor var_624 = slice_by_index(begin = var_624_begin_0, end = var_624_end_0, end_mask = var_624_end_mask_0, x = x_9)[name = tensor("op_624")]; + tensor var_627_begin_0 = const()[name = tensor("op_627_begin_0"), val = tensor([0, 1, 0])]; + tensor var_627_end_0 = const()[name = tensor("op_627_end_0"), val = tensor([1, 16, 256])]; + tensor var_627_end_mask_0 = const()[name = tensor("op_627_end_mask_0"), val = tensor([true, true, true])]; + tensor var_627 = slice_by_index(begin = var_627_begin_0, end = var_627_end_0, end_mask = var_627_end_mask_0, x = window_17)[name = tensor("op_627")]; tensor rolled_15_interleave_0 = const()[name = tensor("rolled_15_interleave_0"), val = tensor(false)]; - tensor rolled_15 = concat(axis = var_34, interleave = rolled_15_interleave_0, values = (var_950, var_947))[name = tensor("rolled_15")]; + tensor rolled_15 = concat(axis = var_34, interleave = rolled_15_interleave_0, values = (var_627, var_624))[name = tensor("rolled_15")]; tensor clip_14 = clip(alpha = var_22, beta = var_21, x = clip_13)[name = tensor("clip_14")]; - tensor var_955 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_955")]; - tensor var_956_dtype_0 = const()[name = tensor("op_956_dtype_0"), val = tensor("fp32")]; - tensor var_957 = const()[name = tensor("op_957"), val = tensor([1, 16, 1])]; - tensor var_956 = cast(dtype = var_956_dtype_0, x = var_955)[name = tensor("cast_177")]; - tensor fill_mask_15 = reshape(shape = var_957, x = var_956)[name = tensor("fill_mask_15")]; + tensor var_632 = equal(x = encoder__conv_positions, y = clip_14)[name = tensor("op_632")]; + tensor var_633_dtype_0 = const()[name = tensor("op_633_dtype_0"), val = tensor("fp32")]; + tensor var_634 = const()[name = tensor("op_634"), val = tensor([1, 16, 1])]; + tensor var_633 = cast(dtype = var_633_dtype_0, x = var_632)[name = tensor("cast_177")]; + tensor fill_mask_15 = reshape(shape = var_634, x = var_633)[name = tensor("fill_mask_15")]; tensor x_expanded_15_reps_0 = const()[name = tensor("x_expanded_15_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_947)[name = tensor("x_expanded_15")]; - tensor var_961 = sub(x = var_18, y = fill_mask_15)[name = tensor("op_961")]; - tensor var_962 = mul(x = input_111, y = var_961)[name = tensor("op_962")]; - tensor var_963 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_963")]; - tensor filled_window_15 = add(x = var_962, y = var_963)[name = tensor("filled_window_15")]; - tensor var_965 = greater_equal(x = clip_13, y = var_17)[name = tensor("op_965")]; - tensor var_966_dtype_0 = const()[name = tensor("op_966_dtype_0"), val = tensor("fp32")]; - tensor var_967 = const()[name = tensor("op_967"), val = tensor([1, 1, 1])]; - tensor var_966 = cast(dtype = var_966_dtype_0, x = var_965)[name = tensor("cast_176")]; - tensor use_roll_15 = reshape(shape = var_967, x = var_966)[name = tensor("use_roll_15")]; - tensor var_969 = sub(x = var_18, y = use_roll_15)[name = tensor("op_969")]; - tensor var_970 = mul(x = filled_window_15, y = var_969)[name = tensor("op_970")]; - tensor var_971 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_971")]; - tensor input_121 = add(x = var_970, y = var_971)[name = tensor("input_121")]; - tensor var_973 = const()[name = tensor("op_973"), val = tensor(0x1p+0)]; - tensor var_974 = add(x = clip_13, y = var_973)[name = tensor("op_974")]; - tensor clip_15 = clip(alpha = var_22, beta = var_17, x = var_974)[name = tensor("clip_15")]; - tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; - tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_121)[name = tensor("x_23")]; - tensor input_123_perm_0 = const()[name = tensor("input_123_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_49_pad_type_0 = const()[name = tensor("inputs_49_pad_type_0"), val = tensor("valid")]; - tensor inputs_49_strides_0 = const()[name = tensor("inputs_49_strides_0"), val = tensor([1])]; - tensor inputs_49_pad_0 = const()[name = tensor("inputs_49_pad_0"), val = tensor([0, 0])]; - tensor inputs_49_dilations_0 = const()[name = tensor("inputs_49_dilations_0"), val = tensor([1])]; - tensor inputs_49_groups_0 = const()[name = tensor("inputs_49_groups_0"), val = tensor(1)]; - tensor input_123 = transpose(perm = input_123_perm_0, x = x_23)[name = tensor("transpose_64")]; - tensor inputs_49 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_49_dilations_0, groups = inputs_49_groups_0, pad = inputs_49_pad_0, pad_type = inputs_49_pad_type_0, strides = inputs_49_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_123)[name = tensor("inputs_49")]; - tensor var_995_split_sizes_0 = const()[name = tensor("op_995_split_sizes_0"), val = tensor([256, 256])]; - tensor var_995_axis_0 = const()[name = tensor("op_995_axis_0"), val = tensor(1)]; - tensor var_995_0, tensor var_995_1 = split(axis = var_995_axis_0, split_sizes = var_995_split_sizes_0, x = inputs_49)[name = tensor("op_995")]; - tensor var_997 = sigmoid(x = var_995_1)[name = tensor("op_997")]; - tensor inputs_51 = mul(x = var_995_0, y = var_997)[name = tensor("inputs_51")]; - tensor outputs_aug_15_pad_type_0 = const()[name = tensor("outputs_aug_15_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_15_pad_0 = const()[name = tensor("outputs_aug_15_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_15_groups_0 = const()[name = tensor("outputs_aug_15_groups_0"), val = tensor(256)]; - tensor outputs_aug_15_strides_0 = const()[name = tensor("outputs_aug_15_strides_0"), val = tensor([1])]; - tensor outputs_aug_15_dilations_0 = const()[name = tensor("outputs_aug_15_dilations_0"), val = tensor([1])]; - tensor outputs_aug_15 = conv(dilations = outputs_aug_15_dilations_0, groups = outputs_aug_15_groups_0, pad = outputs_aug_15_pad_0, pad_type = outputs_aug_15_pad_type_0, strides = outputs_aug_15_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_51)[name = tensor("outputs_aug_15")]; - tensor input_125_begin_0 = const()[name = tensor("input_125_begin_0"), val = tensor([0, 0, 0])]; - tensor input_125_end_0 = const()[name = tensor("input_125_end_0"), val = tensor([1, 256, 16])]; - tensor input_125_end_mask_0 = const()[name = tensor("input_125_end_mask_0"), val = tensor([true, true, false])]; - tensor input_125 = slice_by_index(begin = input_125_begin_0, end = input_125_end_0, end_mask = input_125_end_mask_0, x = outputs_aug_15)[name = tensor("input_125")]; - tensor inputs_53 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_125)[name = tensor("inputs_53")]; - tensor input_127 = silu(x = inputs_53)[name = tensor("input_127")]; - tensor input_129_pad_type_0 = const()[name = tensor("input_129_pad_type_0"), val = tensor("valid")]; - tensor input_129_strides_0 = const()[name = tensor("input_129_strides_0"), val = tensor([1])]; - tensor input_129_pad_0 = const()[name = tensor("input_129_pad_0"), val = tensor([0, 0])]; - tensor input_129_dilations_0 = const()[name = tensor("input_129_dilations_0"), val = tensor([1])]; - tensor input_129_groups_0 = const()[name = tensor("input_129_groups_0"), val = tensor(1)]; - tensor input_129 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_129_dilations_0, groups = input_129_groups_0, pad = input_129_pad_0, pad_type = input_129_pad_type_0, strides = input_129_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_127)[name = tensor("input_129")]; - tensor conv_out_15_perm_0 = const()[name = tensor("conv_out_15_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1028_begin_0 = const()[name = tensor("op_1028_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1028_end_0 = const()[name = tensor("op_1028_end_0"), val = tensor([1, 16, 256])]; - tensor var_1028_end_mask_0 = const()[name = tensor("op_1028_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_15 = transpose(perm = conv_out_15_perm_0, x = input_129)[name = tensor("transpose_63")]; - tensor var_1028 = slice_by_index(begin = var_1028_begin_0, end = var_1028_end_0, end_mask = var_1028_end_mask_0, x = conv_out_15)[name = tensor("op_1028")]; - tensor var_1031_begin_0 = const()[name = tensor("op_1031_begin_0"), val = tensor([0, 3, 0])]; - tensor var_1031_end_0 = const()[name = tensor("op_1031_end_0"), val = tensor([1, 4, 256])]; - tensor var_1031_end_mask_0 = const()[name = tensor("op_1031_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1031 = slice_by_index(begin = var_1031_begin_0, end = var_1031_end_0, end_mask = var_1031_end_mask_0, x = x_17)[name = tensor("op_1031")]; - tensor var_1034_begin_0 = const()[name = tensor("op_1034_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1034_end_0 = const()[name = tensor("op_1034_end_0"), val = tensor([1, 16, 256])]; - tensor var_1034_end_mask_0 = const()[name = tensor("op_1034_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1034 = slice_by_index(begin = var_1034_begin_0, end = var_1034_end_0, end_mask = var_1034_end_mask_0, x = input_121)[name = tensor("op_1034")]; + tensor x_expanded_15 = tile(reps = x_expanded_15_reps_0, x = var_624)[name = tensor("x_expanded_15")]; + tensor var_638 = sub(x = var_18, y = fill_mask_15)[name = tensor("op_638")]; + tensor var_639 = mul(x = window_17, y = var_638)[name = tensor("op_639")]; + tensor var_640 = mul(x = x_expanded_15, y = fill_mask_15)[name = tensor("op_640")]; + tensor filled_window_15 = add(x = var_639, y = var_640)[name = tensor("filled_window_15")]; + tensor var_642 = greater_equal(x = clip_13, y = var_17)[name = tensor("op_642")]; + tensor var_643_dtype_0 = const()[name = tensor("op_643_dtype_0"), val = tensor("fp32")]; + tensor var_644 = const()[name = tensor("op_644"), val = tensor([1, 1, 1])]; + tensor var_643 = cast(dtype = var_643_dtype_0, x = var_642)[name = tensor("cast_176")]; + tensor use_roll_15 = reshape(shape = var_644, x = var_643)[name = tensor("use_roll_15")]; + tensor var_646 = sub(x = var_18, y = use_roll_15)[name = tensor("op_646")]; + tensor var_647 = mul(x = filled_window_15, y = var_646)[name = tensor("op_647")]; + tensor var_648 = mul(x = rolled_15, y = use_roll_15)[name = tensor("op_648")]; + tensor window_19 = add(x = var_647, y = var_648)[name = tensor("window_19")]; + tensor var_650 = const()[name = tensor("op_650"), val = tensor(0x1p+0)]; + tensor var_651 = add(x = clip_13, y = var_650)[name = tensor("op_651")]; + tensor clip_15 = clip(alpha = var_22, beta = var_17, x = var_651)[name = tensor("clip_15")]; + tensor var_654_begin_0 = const()[name = tensor("op_654_begin_0"), val = tensor([0, 3, 0])]; + tensor var_654_end_0 = const()[name = tensor("op_654_end_0"), val = tensor([1, 4, 256])]; + tensor var_654_end_mask_0 = const()[name = tensor("op_654_end_mask_0"), val = tensor([true, false, true])]; + tensor var_654 = slice_by_index(begin = var_654_begin_0, end = var_654_end_0, end_mask = var_654_end_mask_0, x = x_9)[name = tensor("op_654")]; + tensor var_657_begin_0 = const()[name = tensor("op_657_begin_0"), val = tensor([0, 1, 0])]; + tensor var_657_end_0 = const()[name = tensor("op_657_end_0"), val = tensor([1, 16, 256])]; + tensor var_657_end_mask_0 = const()[name = tensor("op_657_end_mask_0"), val = tensor([true, true, true])]; + tensor var_657 = slice_by_index(begin = var_657_begin_0, end = var_657_end_0, end_mask = var_657_end_mask_0, x = window_19)[name = tensor("op_657")]; tensor rolled_17_interleave_0 = const()[name = tensor("rolled_17_interleave_0"), val = tensor(false)]; - tensor rolled_17 = concat(axis = var_34, interleave = rolled_17_interleave_0, values = (var_1034, var_1031))[name = tensor("rolled_17")]; + tensor rolled_17 = concat(axis = var_34, interleave = rolled_17_interleave_0, values = (var_657, var_654))[name = tensor("rolled_17")]; tensor clip_16 = clip(alpha = var_22, beta = var_21, x = clip_15)[name = tensor("clip_16")]; - tensor var_1039 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_1039")]; - tensor var_1040_dtype_0 = const()[name = tensor("op_1040_dtype_0"), val = tensor("fp32")]; - tensor var_1041 = const()[name = tensor("op_1041"), val = tensor([1, 16, 1])]; - tensor var_1040 = cast(dtype = var_1040_dtype_0, x = var_1039)[name = tensor("cast_175")]; - tensor fill_mask_17 = reshape(shape = var_1041, x = var_1040)[name = tensor("fill_mask_17")]; + tensor var_662 = equal(x = encoder__conv_positions, y = clip_16)[name = tensor("op_662")]; + tensor var_663_dtype_0 = const()[name = tensor("op_663_dtype_0"), val = tensor("fp32")]; + tensor var_664 = const()[name = tensor("op_664"), val = tensor([1, 16, 1])]; + tensor var_663 = cast(dtype = var_663_dtype_0, x = var_662)[name = tensor("cast_175")]; + tensor fill_mask_17 = reshape(shape = var_664, x = var_663)[name = tensor("fill_mask_17")]; tensor x_expanded_17_reps_0 = const()[name = tensor("x_expanded_17_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_1031)[name = tensor("x_expanded_17")]; - tensor var_1045 = sub(x = var_18, y = fill_mask_17)[name = tensor("op_1045")]; - tensor var_1046 = mul(x = input_121, y = var_1045)[name = tensor("op_1046")]; - tensor var_1047 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_1047")]; - tensor filled_window_17 = add(x = var_1046, y = var_1047)[name = tensor("filled_window_17")]; - tensor var_1049 = greater_equal(x = clip_15, y = var_17)[name = tensor("op_1049")]; - tensor var_1050_dtype_0 = const()[name = tensor("op_1050_dtype_0"), val = tensor("fp32")]; - tensor var_1051 = const()[name = tensor("op_1051"), val = tensor([1, 1, 1])]; - tensor var_1050 = cast(dtype = var_1050_dtype_0, x = var_1049)[name = tensor("cast_174")]; - tensor use_roll_17 = reshape(shape = var_1051, x = var_1050)[name = tensor("use_roll_17")]; - tensor var_1053 = sub(x = var_18, y = use_roll_17)[name = tensor("op_1053")]; - tensor var_1054 = mul(x = filled_window_17, y = var_1053)[name = tensor("op_1054")]; - tensor var_1055 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_1055")]; - tensor input_131 = add(x = var_1054, y = var_1055)[name = tensor("input_131")]; - tensor var_1057 = const()[name = tensor("op_1057"), val = tensor(0x1p+0)]; - tensor var_1058 = add(x = clip_15, y = var_1057)[name = tensor("op_1058")]; - tensor clip_17 = clip(alpha = var_22, beta = var_17, x = var_1058)[name = tensor("clip_17")]; - tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; - tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_131)[name = tensor("x_25")]; - tensor input_133_perm_0 = const()[name = tensor("input_133_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_55_pad_type_0 = const()[name = tensor("inputs_55_pad_type_0"), val = tensor("valid")]; - tensor inputs_55_strides_0 = const()[name = tensor("inputs_55_strides_0"), val = tensor([1])]; - tensor inputs_55_pad_0 = const()[name = tensor("inputs_55_pad_0"), val = tensor([0, 0])]; - tensor inputs_55_dilations_0 = const()[name = tensor("inputs_55_dilations_0"), val = tensor([1])]; - tensor inputs_55_groups_0 = const()[name = tensor("inputs_55_groups_0"), val = tensor(1)]; - tensor input_133 = transpose(perm = input_133_perm_0, x = x_25)[name = tensor("transpose_62")]; - tensor inputs_55 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_55_dilations_0, groups = inputs_55_groups_0, pad = inputs_55_pad_0, pad_type = inputs_55_pad_type_0, strides = inputs_55_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_133)[name = tensor("inputs_55")]; - tensor var_1079_split_sizes_0 = const()[name = tensor("op_1079_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1079_axis_0 = const()[name = tensor("op_1079_axis_0"), val = tensor(1)]; - tensor var_1079_0, tensor var_1079_1 = split(axis = var_1079_axis_0, split_sizes = var_1079_split_sizes_0, x = inputs_55)[name = tensor("op_1079")]; - tensor var_1081 = sigmoid(x = var_1079_1)[name = tensor("op_1081")]; - tensor inputs_57 = mul(x = var_1079_0, y = var_1081)[name = tensor("inputs_57")]; - tensor outputs_aug_17_pad_type_0 = const()[name = tensor("outputs_aug_17_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_17_pad_0 = const()[name = tensor("outputs_aug_17_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_17_groups_0 = const()[name = tensor("outputs_aug_17_groups_0"), val = tensor(256)]; - tensor outputs_aug_17_strides_0 = const()[name = tensor("outputs_aug_17_strides_0"), val = tensor([1])]; - tensor outputs_aug_17_dilations_0 = const()[name = tensor("outputs_aug_17_dilations_0"), val = tensor([1])]; - tensor outputs_aug_17 = conv(dilations = outputs_aug_17_dilations_0, groups = outputs_aug_17_groups_0, pad = outputs_aug_17_pad_0, pad_type = outputs_aug_17_pad_type_0, strides = outputs_aug_17_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_57)[name = tensor("outputs_aug_17")]; - tensor input_135_begin_0 = const()[name = tensor("input_135_begin_0"), val = tensor([0, 0, 0])]; - tensor input_135_end_0 = const()[name = tensor("input_135_end_0"), val = tensor([1, 256, 16])]; - tensor input_135_end_mask_0 = const()[name = tensor("input_135_end_mask_0"), val = tensor([true, true, false])]; - tensor input_135 = slice_by_index(begin = input_135_begin_0, end = input_135_end_0, end_mask = input_135_end_mask_0, x = outputs_aug_17)[name = tensor("input_135")]; - tensor inputs_59 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_135)[name = tensor("inputs_59")]; - tensor input_137 = silu(x = inputs_59)[name = tensor("input_137")]; - tensor input_139_pad_type_0 = const()[name = tensor("input_139_pad_type_0"), val = tensor("valid")]; - tensor input_139_strides_0 = const()[name = tensor("input_139_strides_0"), val = tensor([1])]; - tensor input_139_pad_0 = const()[name = tensor("input_139_pad_0"), val = tensor([0, 0])]; - tensor input_139_dilations_0 = const()[name = tensor("input_139_dilations_0"), val = tensor([1])]; - tensor input_139_groups_0 = const()[name = tensor("input_139_groups_0"), val = tensor(1)]; - tensor input_139 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_139_dilations_0, groups = input_139_groups_0, pad = input_139_pad_0, pad_type = input_139_pad_type_0, strides = input_139_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_137)[name = tensor("input_139")]; - tensor conv_out_17_perm_0 = const()[name = tensor("conv_out_17_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1112_begin_0 = const()[name = tensor("op_1112_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1112_end_0 = const()[name = tensor("op_1112_end_0"), val = tensor([1, 16, 256])]; - tensor var_1112_end_mask_0 = const()[name = tensor("op_1112_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_17 = transpose(perm = conv_out_17_perm_0, x = input_139)[name = tensor("transpose_61")]; - tensor var_1112 = slice_by_index(begin = var_1112_begin_0, end = var_1112_end_0, end_mask = var_1112_end_mask_0, x = conv_out_17)[name = tensor("op_1112")]; - tensor var_1115_begin_0 = const()[name = tensor("op_1115_begin_0"), val = tensor([0, 4, 0])]; - tensor var_1115_end_0 = const()[name = tensor("op_1115_end_0"), val = tensor([1, 1, 256])]; - tensor var_1115_end_mask_0 = const()[name = tensor("op_1115_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1115 = slice_by_index(begin = var_1115_begin_0, end = var_1115_end_0, end_mask = var_1115_end_mask_0, x = x_17)[name = tensor("op_1115")]; - tensor var_1118_begin_0 = const()[name = tensor("op_1118_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1118_end_0 = const()[name = tensor("op_1118_end_0"), val = tensor([1, 16, 256])]; - tensor var_1118_end_mask_0 = const()[name = tensor("op_1118_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1118 = slice_by_index(begin = var_1118_begin_0, end = var_1118_end_0, end_mask = var_1118_end_mask_0, x = input_131)[name = tensor("op_1118")]; + tensor x_expanded_17 = tile(reps = x_expanded_17_reps_0, x = var_654)[name = tensor("x_expanded_17")]; + tensor var_668 = sub(x = var_18, y = fill_mask_17)[name = tensor("op_668")]; + tensor var_669 = mul(x = window_19, y = var_668)[name = tensor("op_669")]; + tensor var_670 = mul(x = x_expanded_17, y = fill_mask_17)[name = tensor("op_670")]; + tensor filled_window_17 = add(x = var_669, y = var_670)[name = tensor("filled_window_17")]; + tensor var_672 = greater_equal(x = clip_15, y = var_17)[name = tensor("op_672")]; + tensor var_673_dtype_0 = const()[name = tensor("op_673_dtype_0"), val = tensor("fp32")]; + tensor var_674 = const()[name = tensor("op_674"), val = tensor([1, 1, 1])]; + tensor var_673 = cast(dtype = var_673_dtype_0, x = var_672)[name = tensor("cast_174")]; + tensor use_roll_17 = reshape(shape = var_674, x = var_673)[name = tensor("use_roll_17")]; + tensor var_676 = sub(x = var_18, y = use_roll_17)[name = tensor("op_676")]; + tensor var_677 = mul(x = filled_window_17, y = var_676)[name = tensor("op_677")]; + tensor var_678 = mul(x = rolled_17, y = use_roll_17)[name = tensor("op_678")]; + tensor window_21 = add(x = var_677, y = var_678)[name = tensor("window_21")]; + tensor var_680 = const()[name = tensor("op_680"), val = tensor(0x1p+0)]; + tensor var_681 = add(x = clip_15, y = var_680)[name = tensor("op_681")]; + tensor clip_17 = clip(alpha = var_22, beta = var_17, x = var_681)[name = tensor("clip_17")]; + tensor var_684_begin_0 = const()[name = tensor("op_684_begin_0"), val = tensor([0, 4, 0])]; + tensor var_684_end_0 = const()[name = tensor("op_684_end_0"), val = tensor([1, 1, 256])]; + tensor var_684_end_mask_0 = const()[name = tensor("op_684_end_mask_0"), val = tensor([true, true, true])]; + tensor var_684 = slice_by_index(begin = var_684_begin_0, end = var_684_end_0, end_mask = var_684_end_mask_0, x = x_9)[name = tensor("op_684")]; + tensor var_687_begin_0 = const()[name = tensor("op_687_begin_0"), val = tensor([0, 1, 0])]; + tensor var_687_end_0 = const()[name = tensor("op_687_end_0"), val = tensor([1, 16, 256])]; + tensor var_687_end_mask_0 = const()[name = tensor("op_687_end_mask_0"), val = tensor([true, true, true])]; + tensor var_687 = slice_by_index(begin = var_687_begin_0, end = var_687_end_0, end_mask = var_687_end_mask_0, x = window_21)[name = tensor("op_687")]; tensor rolled_19_interleave_0 = const()[name = tensor("rolled_19_interleave_0"), val = tensor(false)]; - tensor rolled_19 = concat(axis = var_34, interleave = rolled_19_interleave_0, values = (var_1118, var_1115))[name = tensor("rolled_19")]; + tensor rolled_19 = concat(axis = var_34, interleave = rolled_19_interleave_0, values = (var_687, var_684))[name = tensor("rolled_19")]; tensor clip_18 = clip(alpha = var_22, beta = var_21, x = clip_17)[name = tensor("clip_18")]; - tensor var_1123 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_1123")]; - tensor var_1124_dtype_0 = const()[name = tensor("op_1124_dtype_0"), val = tensor("fp32")]; - tensor var_1125 = const()[name = tensor("op_1125"), val = tensor([1, 16, 1])]; - tensor var_1124 = cast(dtype = var_1124_dtype_0, x = var_1123)[name = tensor("cast_173")]; - tensor fill_mask_19 = reshape(shape = var_1125, x = var_1124)[name = tensor("fill_mask_19")]; + tensor var_692 = equal(x = encoder__conv_positions, y = clip_18)[name = tensor("op_692")]; + tensor var_693_dtype_0 = const()[name = tensor("op_693_dtype_0"), val = tensor("fp32")]; + tensor var_694 = const()[name = tensor("op_694"), val = tensor([1, 16, 1])]; + tensor var_693 = cast(dtype = var_693_dtype_0, x = var_692)[name = tensor("cast_173")]; + tensor fill_mask_19 = reshape(shape = var_694, x = var_693)[name = tensor("fill_mask_19")]; tensor x_expanded_19_reps_0 = const()[name = tensor("x_expanded_19_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_1115)[name = tensor("x_expanded_19")]; - tensor var_1129 = sub(x = var_18, y = fill_mask_19)[name = tensor("op_1129")]; - tensor var_1130 = mul(x = input_131, y = var_1129)[name = tensor("op_1130")]; - tensor var_1131 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_1131")]; - tensor filled_window_19 = add(x = var_1130, y = var_1131)[name = tensor("filled_window_19")]; - tensor var_1133 = greater_equal(x = clip_17, y = var_17)[name = tensor("op_1133")]; - tensor var_1134_dtype_0 = const()[name = tensor("op_1134_dtype_0"), val = tensor("fp32")]; - tensor var_1135 = const()[name = tensor("op_1135"), val = tensor([1, 1, 1])]; - tensor var_1134 = cast(dtype = var_1134_dtype_0, x = var_1133)[name = tensor("cast_172")]; - tensor use_roll_19 = reshape(shape = var_1135, x = var_1134)[name = tensor("use_roll_19")]; - tensor var_1137 = sub(x = var_18, y = use_roll_19)[name = tensor("op_1137")]; - tensor var_1138 = mul(x = filled_window_19, y = var_1137)[name = tensor("op_1138")]; - tensor var_1139 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_1139")]; - tensor input_141 = add(x = var_1138, y = var_1139)[name = tensor("input_141")]; - tensor var_1141 = const()[name = tensor("op_1141"), val = tensor(0x1p+0)]; - tensor var_1142 = add(x = clip_17, y = var_1141)[name = tensor("op_1142")]; - tensor clip_19 = clip(alpha = var_22, beta = var_17, x = var_1142)[name = tensor("clip_19")]; - tensor x_27_axes_0 = const()[name = tensor("x_27_axes_0"), val = tensor([-1])]; - tensor x_27 = layer_norm(axes = x_27_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_141)[name = tensor("x_27")]; - tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_61_pad_type_0 = const()[name = tensor("inputs_61_pad_type_0"), val = tensor("valid")]; - tensor inputs_61_strides_0 = const()[name = tensor("inputs_61_strides_0"), val = tensor([1])]; - tensor inputs_61_pad_0 = const()[name = tensor("inputs_61_pad_0"), val = tensor([0, 0])]; - tensor inputs_61_dilations_0 = const()[name = tensor("inputs_61_dilations_0"), val = tensor([1])]; - tensor inputs_61_groups_0 = const()[name = tensor("inputs_61_groups_0"), val = tensor(1)]; - tensor input_143 = transpose(perm = input_143_perm_0, x = x_27)[name = tensor("transpose_60")]; - tensor inputs_61 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_61_dilations_0, groups = inputs_61_groups_0, pad = inputs_61_pad_0, pad_type = inputs_61_pad_type_0, strides = inputs_61_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_61")]; - tensor var_1163_split_sizes_0 = const()[name = tensor("op_1163_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1163_axis_0 = const()[name = tensor("op_1163_axis_0"), val = tensor(1)]; - tensor var_1163_0, tensor var_1163_1 = split(axis = var_1163_axis_0, split_sizes = var_1163_split_sizes_0, x = inputs_61)[name = tensor("op_1163")]; - tensor var_1165 = sigmoid(x = var_1163_1)[name = tensor("op_1165")]; - tensor inputs_63 = mul(x = var_1163_0, y = var_1165)[name = tensor("inputs_63")]; - tensor outputs_aug_19_pad_type_0 = const()[name = tensor("outputs_aug_19_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_19_pad_0 = const()[name = tensor("outputs_aug_19_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_19_groups_0 = const()[name = tensor("outputs_aug_19_groups_0"), val = tensor(256)]; - tensor outputs_aug_19_strides_0 = const()[name = tensor("outputs_aug_19_strides_0"), val = tensor([1])]; - tensor outputs_aug_19_dilations_0 = const()[name = tensor("outputs_aug_19_dilations_0"), val = tensor([1])]; - tensor outputs_aug_19 = conv(dilations = outputs_aug_19_dilations_0, groups = outputs_aug_19_groups_0, pad = outputs_aug_19_pad_0, pad_type = outputs_aug_19_pad_type_0, strides = outputs_aug_19_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_63)[name = tensor("outputs_aug_19")]; - tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; - tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([1, 256, 16])]; - tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; - tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug_19)[name = tensor("input_145")]; - tensor inputs_65 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_145)[name = tensor("inputs_65")]; - tensor input_147 = silu(x = inputs_65)[name = tensor("input_147")]; - tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; - tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; - tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; - tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; - tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; - tensor input_149 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; - tensor conv_out_19_perm_0 = const()[name = tensor("conv_out_19_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1196_begin_0 = const()[name = tensor("op_1196_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1196_end_0 = const()[name = tensor("op_1196_end_0"), val = tensor([1, 16, 256])]; - tensor var_1196_end_mask_0 = const()[name = tensor("op_1196_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_19 = transpose(perm = conv_out_19_perm_0, x = input_149)[name = tensor("transpose_59")]; - tensor var_1196 = slice_by_index(begin = var_1196_begin_0, end = var_1196_end_0, end_mask = var_1196_end_mask_0, x = conv_out_19)[name = tensor("op_1196")]; - tensor var_1199_interleave_0 = const()[name = tensor("op_1199_interleave_0"), val = tensor(false)]; - tensor var_1199 = concat(axis = var_34, interleave = var_1199_interleave_0, values = (var_860, var_944, var_1028, var_1112, var_1196))[name = tensor("op_1199")]; - tensor input_151 = add(x = x_17, y = var_1199)[name = tensor("input_151")]; - tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; - tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; - tensor inputs_67 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_17")]; - tensor input_155 = silu(x = inputs_67)[name = tensor("input_155")]; - tensor input_159 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_18")]; - tensor var_1222 = const()[name = tensor("op_1222"), val = tensor(0x1p-1)]; - tensor var_1223 = mul(x = input_159, y = var_1222)[name = tensor("op_1223")]; - tensor input_161 = add(x = var_1223, y = input_151)[name = tensor("input_161")]; - tensor input_163_axes_0 = const()[name = tensor("input_163_axes_0"), val = tensor([-1])]; - tensor input_163 = layer_norm(axes = input_163_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_36, gamma = encoder_layer_norm_1_weight, x = input_161)[name = tensor("input_163")]; - tensor input_165_axes_0 = const()[name = tensor("input_165_axes_0"), val = tensor([-1])]; - tensor input_165 = layer_norm(axes = input_165_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_163)[name = tensor("input_165")]; - tensor inputs_69 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_165)[name = tensor("linear_19")]; - tensor input_167 = silu(x = inputs_69)[name = tensor("input_167")]; - tensor input_171 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_167)[name = tensor("linear_20")]; - tensor var_1252 = const()[name = tensor("op_1252"), val = tensor(0x1p-1)]; - tensor var_1253 = mul(x = input_171, y = var_1252)[name = tensor("op_1253")]; - tensor input_173 = add(x = var_1253, y = input_163)[name = tensor("input_173")]; - tensor x_29_axes_0 = const()[name = tensor("x_29_axes_0"), val = tensor([-1])]; - tensor x_29 = layer_norm(axes = x_29_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_36, gamma = encoder_ret_lns_2_weight, x = input_173)[name = tensor("x_29")]; + tensor x_expanded_19 = tile(reps = x_expanded_19_reps_0, x = var_684)[name = tensor("x_expanded_19")]; + tensor var_698 = sub(x = var_18, y = fill_mask_19)[name = tensor("op_698")]; + tensor var_699 = mul(x = window_21, y = var_698)[name = tensor("op_699")]; + tensor var_700 = mul(x = x_expanded_19, y = fill_mask_19)[name = tensor("op_700")]; + tensor filled_window_19 = add(x = var_699, y = var_700)[name = tensor("filled_window_19")]; + tensor var_702 = greater_equal(x = clip_17, y = var_17)[name = tensor("op_702")]; + tensor var_703_dtype_0 = const()[name = tensor("op_703_dtype_0"), val = tensor("fp32")]; + tensor var_704 = const()[name = tensor("op_704"), val = tensor([1, 1, 1])]; + tensor var_703 = cast(dtype = var_703_dtype_0, x = var_702)[name = tensor("cast_172")]; + tensor use_roll_19 = reshape(shape = var_704, x = var_703)[name = tensor("use_roll_19")]; + tensor var_706 = sub(x = var_18, y = use_roll_19)[name = tensor("op_706")]; + tensor var_707 = mul(x = filled_window_19, y = var_706)[name = tensor("op_707")]; + tensor var_708 = mul(x = rolled_19, y = use_roll_19)[name = tensor("op_708")]; + tensor window_23 = add(x = var_707, y = var_708)[name = tensor("window_23")]; + tensor var_710 = const()[name = tensor("op_710"), val = tensor(0x1p+0)]; + tensor var_711 = add(x = clip_17, y = var_710)[name = tensor("op_711")]; + tensor clip_19 = clip(alpha = var_22, beta = var_17, x = var_711)[name = tensor("clip_19")]; + tensor input_61_interleave_0 = const()[name = tensor("input_61_interleave_0"), val = tensor(false)]; + tensor input_61 = concat(axis = var_31, interleave = input_61_interleave_0, values = (window_15, window_17, window_19, window_21, window_23))[name = tensor("input_61")]; + tensor x_11_axes_0 = const()[name = tensor("x_11_axes_0"), val = tensor([-1])]; + tensor x_11 = layer_norm(axes = x_11_axes_0, beta = encoder_conv_module_1_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_0_weight, x = input_61)[name = tensor("x_11")]; + tensor input_63_perm_0 = const()[name = tensor("input_63_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_13_pad_type_0 = const()[name = tensor("inputs_13_pad_type_0"), val = tensor("valid")]; + tensor inputs_13_strides_0 = const()[name = tensor("inputs_13_strides_0"), val = tensor([1])]; + tensor inputs_13_pad_0 = const()[name = tensor("inputs_13_pad_0"), val = tensor([0, 0])]; + tensor inputs_13_dilations_0 = const()[name = tensor("inputs_13_dilations_0"), val = tensor([1])]; + tensor inputs_13_groups_0 = const()[name = tensor("inputs_13_groups_0"), val = tensor(1)]; + tensor input_63 = transpose(perm = input_63_perm_0, x = x_11)[name = tensor("transpose_47")]; + tensor inputs_13 = conv(bias = encoder_conv_module_1_sequential_2_conv_bias, dilations = inputs_13_dilations_0, groups = inputs_13_groups_0, pad = inputs_13_pad_0, pad_type = inputs_13_pad_type_0, strides = inputs_13_strides_0, weight = encoder_conv_module_1_sequential_2_conv_weight, x = input_63)[name = tensor("inputs_13")]; + tensor var_734_split_sizes_0 = const()[name = tensor("op_734_split_sizes_0"), val = tensor([256, 256])]; + tensor var_734_axis_0 = const()[name = tensor("op_734_axis_0"), val = tensor(1)]; + tensor var_734_0, tensor var_734_1 = split(axis = var_734_axis_0, split_sizes = var_734_split_sizes_0, x = inputs_13)[name = tensor("op_734")]; + tensor var_736 = sigmoid(x = var_734_1)[name = tensor("op_736")]; + tensor inputs_15 = mul(x = var_734_0, y = var_736)[name = tensor("inputs_15")]; + tensor outputs_aug_3_pad_type_0 = const()[name = tensor("outputs_aug_3_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_3_pad_0 = const()[name = tensor("outputs_aug_3_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_3_groups_0 = const()[name = tensor("outputs_aug_3_groups_0"), val = tensor(256)]; + tensor outputs_aug_3_strides_0 = const()[name = tensor("outputs_aug_3_strides_0"), val = tensor([1])]; + tensor outputs_aug_3_dilations_0 = const()[name = tensor("outputs_aug_3_dilations_0"), val = tensor([1])]; + tensor outputs_aug_3 = conv(dilations = outputs_aug_3_dilations_0, groups = outputs_aug_3_groups_0, pad = outputs_aug_3_pad_0, pad_type = outputs_aug_3_pad_type_0, strides = outputs_aug_3_strides_0, weight = encoder_conv_module_1_sequential_4_conv_weight, x = inputs_15)[name = tensor("outputs_aug_3")]; + tensor input_65_begin_0 = const()[name = tensor("input_65_begin_0"), val = tensor([0, 0, 0])]; + tensor input_65_end_0 = const()[name = tensor("input_65_end_0"), val = tensor([5, 256, 16])]; + tensor input_65_end_mask_0 = const()[name = tensor("input_65_end_mask_0"), val = tensor([true, true, false])]; + tensor input_65 = slice_by_index(begin = input_65_begin_0, end = input_65_end_0, end_mask = input_65_end_mask_0, x = outputs_aug_3)[name = tensor("input_65")]; + tensor inputs_17 = batch_norm(beta = encoder_conv_module_1_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_1_sequential_5_weight, mean = encoder_conv_module_1_sequential_5_running_mean, variance = encoder_conv_module_1_sequential_5_running_var, x = input_65)[name = tensor("inputs_17")]; + tensor input_67 = silu(x = inputs_17)[name = tensor("input_67")]; + tensor input_69_pad_type_0 = const()[name = tensor("input_69_pad_type_0"), val = tensor("valid")]; + tensor input_69_strides_0 = const()[name = tensor("input_69_strides_0"), val = tensor([1])]; + tensor input_69_pad_0 = const()[name = tensor("input_69_pad_0"), val = tensor([0, 0])]; + tensor input_69_dilations_0 = const()[name = tensor("input_69_dilations_0"), val = tensor([1])]; + tensor input_69_groups_0 = const()[name = tensor("input_69_groups_0"), val = tensor(1)]; + tensor input_69 = conv(bias = encoder_conv_module_1_sequential_7_conv_bias, dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = encoder_conv_module_1_sequential_7_conv_weight, x = input_67)[name = tensor("input_69")]; + tensor conv_out_3_perm_0 = const()[name = tensor("conv_out_3_perm_0"), val = tensor([0, 2, 1])]; + tensor var_767_begin_0 = const()[name = tensor("op_767_begin_0"), val = tensor([0, -1, 0])]; + tensor var_767_end_0 = const()[name = tensor("op_767_end_0"), val = tensor([5, 16, 256])]; + tensor var_767_end_mask_0 = const()[name = tensor("op_767_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_3 = transpose(perm = conv_out_3_perm_0, x = input_69)[name = tensor("transpose_46")]; + tensor var_767 = slice_by_index(begin = var_767_begin_0, end = var_767_end_0, end_mask = var_767_end_mask_0, x = conv_out_3)[name = tensor("op_767")]; + tensor var_769_perm_0 = const()[name = tensor("op_769_perm_0"), val = tensor([1, 0, 2])]; + tensor var_769 = transpose(perm = var_769_perm_0, x = var_767)[name = tensor("transpose_45")]; + tensor input_71 = add(x = x_9, y = var_769)[name = tensor("input_71")]; + tensor input_73_axes_0 = const()[name = tensor("input_73_axes_0"), val = tensor([-1])]; + tensor input_73 = layer_norm(axes = input_73_axes_0, beta = encoder_ffn2_1_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_1_module_sequential_0_weight, x = input_71)[name = tensor("input_73")]; + tensor inputs_19 = linear(bias = encoder_ffn2_1_module_sequential_1_linear_bias, weight = encoder_ffn2_1_module_sequential_1_linear_weight, x = input_73)[name = tensor("linear_17")]; + tensor input_75 = silu(x = inputs_19)[name = tensor("input_75")]; + tensor input_79 = linear(bias = encoder_ffn2_1_module_sequential_4_linear_bias, weight = encoder_ffn2_1_module_sequential_4_linear_weight, x = input_75)[name = tensor("linear_18")]; + tensor var_792 = const()[name = tensor("op_792"), val = tensor(0x1p-1)]; + tensor var_793 = mul(x = input_79, y = var_792)[name = tensor("op_793")]; + tensor input_81 = add(x = var_793, y = input_71)[name = tensor("input_81")]; + tensor input_83_axes_0 = const()[name = tensor("input_83_axes_0"), val = tensor([-1])]; + tensor input_83 = layer_norm(axes = input_83_axes_0, beta = encoder_layer_norm_1_bias, epsilon = var_36, gamma = encoder_layer_norm_1_weight, x = input_81)[name = tensor("input_83")]; + tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; + tensor input_85 = layer_norm(axes = input_85_axes_0, beta = encoder_ffn1_2_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_2_module_sequential_0_weight, x = input_83)[name = tensor("input_85")]; + tensor inputs_21 = linear(bias = encoder_ffn1_2_module_sequential_1_linear_bias, weight = encoder_ffn1_2_module_sequential_1_linear_weight, x = input_85)[name = tensor("linear_19")]; + tensor input_87 = silu(x = inputs_21)[name = tensor("input_87")]; + tensor input_91 = linear(bias = encoder_ffn1_2_module_sequential_4_linear_bias, weight = encoder_ffn1_2_module_sequential_4_linear_weight, x = input_87)[name = tensor("linear_20")]; + tensor var_822 = const()[name = tensor("op_822"), val = tensor(0x1p-1)]; + tensor var_823 = mul(x = input_91, y = var_822)[name = tensor("op_823")]; + tensor input_93 = add(x = var_823, y = input_83)[name = tensor("input_93")]; + tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; + tensor x_13 = layer_norm(axes = x_13_axes_0, beta = encoder_ret_lns_2_bias, epsilon = var_36, gamma = encoder_ret_lns_2_weight, x = input_93)[name = tensor("x_13")]; tensor prev_kv_5_begin_0 = const()[name = tensor("prev_kv_5_begin_0"), val = tensor([2, 0, 0, 0, 0])]; tensor prev_kv_5_end_0 = const()[name = tensor("prev_kv_5_end_0"), val = tensor([3, 1, 4, 64, 64])]; tensor prev_kv_5_end_mask_0 = const()[name = tensor("prev_kv_5_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1175,462 +868,308 @@ program(1.0) tensor prev_scale_5_end_mask_0 = const()[name = tensor("prev_scale_5_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_5_squeeze_mask_0 = const()[name = tensor("prev_scale_5_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_5 = slice_by_index(begin = prev_scale_5_begin_0, end = prev_scale_5_end_0, end_mask = prev_scale_5_end_mask_0, squeeze_mask = prev_scale_5_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_5")]; - tensor var_1267 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_29)[name = tensor("linear_21")]; - tensor var_1268 = const()[name = tensor("op_1268"), val = tensor([1, 5, 4, 64])]; - tensor var_1269 = reshape(shape = var_1268, x = var_1267)[name = tensor("op_1269")]; + tensor var_837 = linear(bias = encoder_q_proj_2_bias, weight = encoder_q_proj_2_weight, x = x_13)[name = tensor("linear_21")]; + tensor var_838 = const()[name = tensor("op_838"), val = tensor([1, 5, 4, 64])]; + tensor var_839 = reshape(shape = var_838, x = var_837)[name = tensor("op_839")]; tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1273 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_29)[name = tensor("linear_22")]; - tensor var_1274 = const()[name = tensor("op_1274"), val = tensor(0x1p-3)]; - tensor var_1275 = mul(x = var_1273, y = var_1274)[name = tensor("op_1275")]; - tensor var_1276 = const()[name = tensor("op_1276"), val = tensor([1, 5, 4, 64])]; - tensor var_1277 = reshape(shape = var_1276, x = var_1275)[name = tensor("op_1277")]; + tensor var_843 = linear(bias = encoder_k_proj_2_bias, weight = encoder_k_proj_2_weight, x = x_13)[name = tensor("linear_22")]; + tensor var_844 = const()[name = tensor("op_844"), val = tensor(0x1p-3)]; + tensor var_845 = mul(x = var_843, y = var_844)[name = tensor("op_845")]; + tensor var_846 = const()[name = tensor("op_846"), val = tensor([1, 5, 4, 64])]; + tensor var_847 = reshape(shape = var_846, x = var_845)[name = tensor("op_847")]; tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1281 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_29)[name = tensor("linear_23")]; - tensor var_1282 = const()[name = tensor("op_1282"), val = tensor([1, 5, 4, 64])]; - tensor var_1283 = reshape(shape = var_1282, x = var_1281)[name = tensor("op_1283")]; + tensor var_851 = linear(bias = encoder_v_proj_2_bias, weight = encoder_v_proj_2_weight, x = x_13)[name = tensor("linear_23")]; + tensor var_852 = const()[name = tensor("op_852"), val = tensor([1, 5, 4, 64])]; + tensor var_853 = reshape(shape = var_852, x = var_851)[name = tensor("op_853")]; tensor v_5_perm_0 = const()[name = tensor("v_5_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_177 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_29)[name = tensor("linear_24")]; + tensor input_97 = linear(bias = encoder_g_proj_2_bias, weight = encoder_g_proj_2_weight, x = x_13)[name = tensor("linear_24")]; tensor sqrt_s0_5 = sqrt(x = prev_scale_5)[name = tensor("sqrt_s0_5")]; tensor s_t_5 = add(x = prev_scale_5, y = encoder__t_index)[name = tensor("s_t_5")]; tensor sqrt_s_t_5 = sqrt(x = s_t_5)[name = tensor("sqrt_s_t_5")]; tensor qk_5_transpose_x_1 = const()[name = tensor("qk_5_transpose_x_1"), val = tensor(false)]; tensor qk_5_transpose_y_1 = const()[name = tensor("qk_5_transpose_y_1"), val = tensor(true)]; - tensor k_5 = transpose(perm = k_5_perm_0, x = var_1277)[name = tensor("transpose_57")]; - tensor q_5 = transpose(perm = q_5_perm_0, x = var_1269)[name = tensor("transpose_58")]; + tensor k_5 = transpose(perm = k_5_perm_0, x = var_847)[name = tensor("transpose_43")]; + tensor q_5 = transpose(perm = q_5_perm_0, x = var_839)[name = tensor("transpose_44")]; tensor qk_5 = matmul(transpose_x = qk_5_transpose_x_1, transpose_y = qk_5_transpose_y_1, x = q_5, y = k_5)[name = tensor("qk_5")]; - tensor var_1293 = const()[name = tensor("op_1293"), val = tensor([5, 1])]; - tensor var_1294 = reshape(shape = var_1293, x = sqrt_s_t_5)[name = tensor("op_1294")]; - tensor M_5 = real_div(x = encoder__causal_mask, y = var_1294)[name = tensor("M_5")]; - tensor var_1296 = mul(x = qk_5, y = M_5)[name = tensor("op_1296")]; + tensor var_863 = const()[name = tensor("op_863"), val = tensor([5, 1])]; + tensor var_864 = reshape(shape = var_863, x = sqrt_s_t_5)[name = tensor("op_864")]; + tensor M_5 = real_div(x = encoder__causal_mask, y = var_864)[name = tensor("M_5")]; + tensor var_866 = mul(x = qk_5, y = M_5)[name = tensor("op_866")]; tensor inner_5_transpose_x_0 = const()[name = tensor("inner_5_transpose_x_0"), val = tensor(false)]; tensor inner_5_transpose_y_0 = const()[name = tensor("inner_5_transpose_y_0"), val = tensor(false)]; - tensor v_5 = transpose(perm = v_5_perm_0, x = var_1283)[name = tensor("transpose_56")]; - tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_1296, y = v_5)[name = tensor("inner_5")]; - tensor var_1298_transpose_x_0 = const()[name = tensor("op_1298_transpose_x_0"), val = tensor(false)]; - tensor var_1298_transpose_y_0 = const()[name = tensor("op_1298_transpose_y_0"), val = tensor(false)]; - tensor var_1298 = matmul(transpose_x = var_1298_transpose_x_0, transpose_y = var_1298_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_1298")]; - tensor var_1299 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_1299")]; - tensor var_1300 = const()[name = tensor("op_1300"), val = tensor([1, 1, 5, 1])]; - tensor var_1301 = reshape(shape = var_1300, x = var_1299)[name = tensor("op_1301")]; - tensor cross_5 = mul(x = var_1298, y = var_1301)[name = tensor("cross_5")]; + tensor v_5 = transpose(perm = v_5_perm_0, x = var_853)[name = tensor("transpose_42")]; + tensor inner_5 = matmul(transpose_x = inner_5_transpose_x_0, transpose_y = inner_5_transpose_y_0, x = var_866, y = v_5)[name = tensor("inner_5")]; + tensor var_868_transpose_x_0 = const()[name = tensor("op_868_transpose_x_0"), val = tensor(false)]; + tensor var_868_transpose_y_0 = const()[name = tensor("op_868_transpose_y_0"), val = tensor(false)]; + tensor var_868 = matmul(transpose_x = var_868_transpose_x_0, transpose_y = var_868_transpose_y_0, x = q_5, y = prev_kv_5)[name = tensor("op_868")]; + tensor var_869 = real_div(x = sqrt_s0_5, y = sqrt_s_t_5)[name = tensor("op_869")]; + tensor var_870 = const()[name = tensor("op_870"), val = tensor([1, 1, 5, 1])]; + tensor var_871 = reshape(shape = var_870, x = var_869)[name = tensor("op_871")]; + tensor cross_5 = mul(x = var_868, y = var_871)[name = tensor("cross_5")]; tensor out_13 = add(x = inner_5, y = cross_5)[name = tensor("out_13")]; - tensor var_1304 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_1304")]; - tensor var_1306_transpose_x_1 = const()[name = tensor("op_1306_transpose_x_1"), val = tensor(true)]; - tensor var_1306_transpose_y_1 = const()[name = tensor("op_1306_transpose_y_1"), val = tensor(false)]; - tensor var_1306 = matmul(transpose_x = var_1306_transpose_x_1, transpose_y = var_1306_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_1306")]; - tensor new_kv_unnorm_5 = add(x = var_1304, y = var_1306)[name = tensor("new_kv_unnorm_5")]; - tensor var_1308 = const()[name = tensor("op_1308"), val = tensor(0x1.4p+2)]; - tensor new_scale_5 = add(x = prev_scale_5, y = var_1308)[name = tensor("new_scale_5")]; - tensor var_1310 = sqrt(x = new_scale_5)[name = tensor("op_1310")]; - tensor var_1311 = real_div(x = new_kv_unnorm_5, y = var_1310)[name = tensor("op_1311")]; - tensor var_1312_perm_0 = const()[name = tensor("op_1312_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_874 = mul(x = prev_kv_5, y = sqrt_s0_5)[name = tensor("op_874")]; + tensor var_876_transpose_x_1 = const()[name = tensor("op_876_transpose_x_1"), val = tensor(true)]; + tensor var_876_transpose_y_1 = const()[name = tensor("op_876_transpose_y_1"), val = tensor(false)]; + tensor var_876 = matmul(transpose_x = var_876_transpose_x_1, transpose_y = var_876_transpose_y_1, x = k_5, y = v_5)[name = tensor("op_876")]; + tensor new_kv_unnorm_5 = add(x = var_874, y = var_876)[name = tensor("new_kv_unnorm_5")]; + tensor var_878 = const()[name = tensor("op_878"), val = tensor(0x1.4p+2)]; + tensor new_scale_5 = add(x = prev_scale_5, y = var_878)[name = tensor("new_scale_5")]; + tensor var_880 = sqrt(x = new_scale_5)[name = tensor("op_880")]; + tensor var_881 = real_div(x = new_kv_unnorm_5, y = var_880)[name = tensor("op_881")]; + tensor var_882_perm_0 = const()[name = tensor("op_882_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_15_axes_0 = const()[name = tensor("out_15_axes_0"), val = tensor([-1])]; - tensor var_1312 = transpose(perm = var_1312_perm_0, x = out_13)[name = tensor("transpose_55")]; - tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_25, x = var_1312)[name = tensor("out_15")]; - tensor var_1316 = const()[name = tensor("op_1316"), val = tensor([1, 5, 256])]; - tensor out_17 = reshape(shape = var_1316, x = out_15)[name = tensor("out_17")]; - tensor var_1318 = silu(x = input_177)[name = tensor("op_1318")]; - tensor input_179 = mul(x = var_1318, y = out_17)[name = tensor("input_179")]; - tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_179)[name = tensor("linear_25")]; - tensor x_31 = add(x = input_173, y = ret_out_5)[name = tensor("x_31")]; - tensor window_5_begin_0 = const()[name = tensor("window_5_begin_0"), val = tensor([2, 0, 0, 0])]; - tensor window_5_end_0 = const()[name = tensor("window_5_end_0"), val = tensor([3, 1, 16, 256])]; - tensor window_5_end_mask_0 = const()[name = tensor("window_5_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_5_squeeze_mask_0 = const()[name = tensor("window_5_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window_5 = slice_by_index(begin = window_5_begin_0, end = window_5_end_0, end_mask = window_5_end_mask_0, squeeze_mask = window_5_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_5")]; + tensor var_882 = transpose(perm = var_882_perm_0, x = out_13)[name = tensor("transpose_41")]; + tensor out_15 = layer_norm(axes = out_15_axes_0, epsilon = var_25, x = var_882)[name = tensor("out_15")]; + tensor var_886 = const()[name = tensor("op_886"), val = tensor([1, 5, 256])]; + tensor out_17 = reshape(shape = var_886, x = out_15)[name = tensor("out_17")]; + tensor var_888 = silu(x = input_97)[name = tensor("op_888")]; + tensor input_99 = mul(x = var_888, y = out_17)[name = tensor("input_99")]; + tensor ret_out_5 = linear(bias = encoder_out_proj_2_bias, weight = encoder_out_proj_2_weight, x = input_99)[name = tensor("linear_25")]; + tensor x_15 = add(x = input_93, y = ret_out_5)[name = tensor("x_15")]; + tensor window_25_begin_0 = const()[name = tensor("window_25_begin_0"), val = tensor([2, 0, 0, 0])]; + tensor window_25_end_0 = const()[name = tensor("window_25_end_0"), val = tensor([3, 1, 16, 256])]; + tensor window_25_end_mask_0 = const()[name = tensor("window_25_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_25_squeeze_mask_0 = const()[name = tensor("window_25_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_25 = slice_by_index(begin = window_25_begin_0, end = window_25_end_0, end_mask = window_25_end_mask_0, squeeze_mask = window_25_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_25")]; tensor filled_21_begin_0 = const()[name = tensor("filled_21_begin_0"), val = tensor([2, 0])]; tensor filled_21_end_0 = const()[name = tensor("filled_21_end_0"), val = tensor([3, 1])]; tensor filled_21_end_mask_0 = const()[name = tensor("filled_21_end_mask_0"), val = tensor([false, true])]; tensor filled_21_squeeze_mask_0 = const()[name = tensor("filled_21_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_21 = slice_by_index(begin = filled_21_begin_0, end = filled_21_end_0, end_mask = filled_21_end_mask_0, squeeze_mask = filled_21_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_21")]; - tensor var_1327_begin_0 = const()[name = tensor("op_1327_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1327_end_0 = const()[name = tensor("op_1327_end_0"), val = tensor([1, 1, 256])]; - tensor var_1327_end_mask_0 = const()[name = tensor("op_1327_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1327 = slice_by_index(begin = var_1327_begin_0, end = var_1327_end_0, end_mask = var_1327_end_mask_0, x = x_31)[name = tensor("op_1327")]; - tensor var_1330_begin_0 = const()[name = tensor("op_1330_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1330_end_0 = const()[name = tensor("op_1330_end_0"), val = tensor([1, 16, 256])]; - tensor var_1330_end_mask_0 = const()[name = tensor("op_1330_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1330 = slice_by_index(begin = var_1330_begin_0, end = var_1330_end_0, end_mask = var_1330_end_mask_0, x = window_5)[name = tensor("op_1330")]; + tensor var_897_begin_0 = const()[name = tensor("op_897_begin_0"), val = tensor([0, 0, 0])]; + tensor var_897_end_0 = const()[name = tensor("op_897_end_0"), val = tensor([1, 1, 256])]; + tensor var_897_end_mask_0 = const()[name = tensor("op_897_end_mask_0"), val = tensor([true, false, true])]; + tensor var_897 = slice_by_index(begin = var_897_begin_0, end = var_897_end_0, end_mask = var_897_end_mask_0, x = x_15)[name = tensor("op_897")]; + tensor var_900_begin_0 = const()[name = tensor("op_900_begin_0"), val = tensor([0, 1, 0])]; + tensor var_900_end_0 = const()[name = tensor("op_900_end_0"), val = tensor([1, 16, 256])]; + tensor var_900_end_mask_0 = const()[name = tensor("op_900_end_mask_0"), val = tensor([true, true, true])]; + tensor var_900 = slice_by_index(begin = var_900_begin_0, end = var_900_end_0, end_mask = var_900_end_mask_0, x = window_25)[name = tensor("op_900")]; tensor rolled_21_interleave_0 = const()[name = tensor("rolled_21_interleave_0"), val = tensor(false)]; - tensor rolled_21 = concat(axis = var_34, interleave = rolled_21_interleave_0, values = (var_1330, var_1327))[name = tensor("rolled_21")]; + tensor rolled_21 = concat(axis = var_34, interleave = rolled_21_interleave_0, values = (var_900, var_897))[name = tensor("rolled_21")]; tensor clip_20 = clip(alpha = var_22, beta = var_21, x = filled_21)[name = tensor("clip_20")]; - tensor var_1335 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_1335")]; - tensor var_1336_dtype_0 = const()[name = tensor("op_1336_dtype_0"), val = tensor("fp32")]; - tensor var_1337 = const()[name = tensor("op_1337"), val = tensor([1, 16, 1])]; - tensor var_1336 = cast(dtype = var_1336_dtype_0, x = var_1335)[name = tensor("cast_171")]; - tensor fill_mask_21 = reshape(shape = var_1337, x = var_1336)[name = tensor("fill_mask_21")]; + tensor var_905 = equal(x = encoder__conv_positions, y = clip_20)[name = tensor("op_905")]; + tensor var_906_dtype_0 = const()[name = tensor("op_906_dtype_0"), val = tensor("fp32")]; + tensor var_907 = const()[name = tensor("op_907"), val = tensor([1, 16, 1])]; + tensor var_906 = cast(dtype = var_906_dtype_0, x = var_905)[name = tensor("cast_171")]; + tensor fill_mask_21 = reshape(shape = var_907, x = var_906)[name = tensor("fill_mask_21")]; tensor x_expanded_21_reps_0 = const()[name = tensor("x_expanded_21_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_1327)[name = tensor("x_expanded_21")]; - tensor var_1341 = sub(x = var_18, y = fill_mask_21)[name = tensor("op_1341")]; - tensor var_1342 = mul(x = window_5, y = var_1341)[name = tensor("op_1342")]; - tensor var_1343 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_1343")]; - tensor filled_window_21 = add(x = var_1342, y = var_1343)[name = tensor("filled_window_21")]; - tensor var_1345 = greater_equal(x = filled_21, y = var_17)[name = tensor("op_1345")]; - tensor var_1346_dtype_0 = const()[name = tensor("op_1346_dtype_0"), val = tensor("fp32")]; - tensor var_1347 = const()[name = tensor("op_1347"), val = tensor([1, 1, 1])]; - tensor var_1346 = cast(dtype = var_1346_dtype_0, x = var_1345)[name = tensor("cast_170")]; - tensor use_roll_21 = reshape(shape = var_1347, x = var_1346)[name = tensor("use_roll_21")]; - tensor var_1349 = sub(x = var_18, y = use_roll_21)[name = tensor("op_1349")]; - tensor var_1350 = mul(x = filled_window_21, y = var_1349)[name = tensor("op_1350")]; - tensor var_1351 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_1351")]; - tensor input_181 = add(x = var_1350, y = var_1351)[name = tensor("input_181")]; - tensor var_1353 = const()[name = tensor("op_1353"), val = tensor(0x1p+0)]; - tensor var_1354 = add(x = filled_21, y = var_1353)[name = tensor("op_1354")]; - tensor clip_21 = clip(alpha = var_22, beta = var_17, x = var_1354)[name = tensor("clip_21")]; - tensor x_33_axes_0 = const()[name = tensor("x_33_axes_0"), val = tensor([-1])]; - tensor x_33 = layer_norm(axes = x_33_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_181)[name = tensor("x_33")]; - tensor input_183_perm_0 = const()[name = tensor("input_183_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_71_pad_type_0 = const()[name = tensor("inputs_71_pad_type_0"), val = tensor("valid")]; - tensor inputs_71_strides_0 = const()[name = tensor("inputs_71_strides_0"), val = tensor([1])]; - tensor inputs_71_pad_0 = const()[name = tensor("inputs_71_pad_0"), val = tensor([0, 0])]; - tensor inputs_71_dilations_0 = const()[name = tensor("inputs_71_dilations_0"), val = tensor([1])]; - tensor inputs_71_groups_0 = const()[name = tensor("inputs_71_groups_0"), val = tensor(1)]; - tensor input_183 = transpose(perm = input_183_perm_0, x = x_33)[name = tensor("transpose_54")]; - tensor inputs_71 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_71_dilations_0, groups = inputs_71_groups_0, pad = inputs_71_pad_0, pad_type = inputs_71_pad_type_0, strides = inputs_71_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_183)[name = tensor("inputs_71")]; - tensor var_1375_split_sizes_0 = const()[name = tensor("op_1375_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1375_axis_0 = const()[name = tensor("op_1375_axis_0"), val = tensor(1)]; - tensor var_1375_0, tensor var_1375_1 = split(axis = var_1375_axis_0, split_sizes = var_1375_split_sizes_0, x = inputs_71)[name = tensor("op_1375")]; - tensor var_1377 = sigmoid(x = var_1375_1)[name = tensor("op_1377")]; - tensor inputs_73 = mul(x = var_1375_0, y = var_1377)[name = tensor("inputs_73")]; - tensor outputs_aug_21_pad_type_0 = const()[name = tensor("outputs_aug_21_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_21_pad_0 = const()[name = tensor("outputs_aug_21_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_21_groups_0 = const()[name = tensor("outputs_aug_21_groups_0"), val = tensor(256)]; - tensor outputs_aug_21_strides_0 = const()[name = tensor("outputs_aug_21_strides_0"), val = tensor([1])]; - tensor outputs_aug_21_dilations_0 = const()[name = tensor("outputs_aug_21_dilations_0"), val = tensor([1])]; - tensor outputs_aug_21 = conv(dilations = outputs_aug_21_dilations_0, groups = outputs_aug_21_groups_0, pad = outputs_aug_21_pad_0, pad_type = outputs_aug_21_pad_type_0, strides = outputs_aug_21_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_73)[name = tensor("outputs_aug_21")]; - tensor input_185_begin_0 = const()[name = tensor("input_185_begin_0"), val = tensor([0, 0, 0])]; - tensor input_185_end_0 = const()[name = tensor("input_185_end_0"), val = tensor([1, 256, 16])]; - tensor input_185_end_mask_0 = const()[name = tensor("input_185_end_mask_0"), val = tensor([true, true, false])]; - tensor input_185 = slice_by_index(begin = input_185_begin_0, end = input_185_end_0, end_mask = input_185_end_mask_0, x = outputs_aug_21)[name = tensor("input_185")]; - tensor inputs_75 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_185)[name = tensor("inputs_75")]; - tensor input_187 = silu(x = inputs_75)[name = tensor("input_187")]; - tensor input_189_pad_type_0 = const()[name = tensor("input_189_pad_type_0"), val = tensor("valid")]; - tensor input_189_strides_0 = const()[name = tensor("input_189_strides_0"), val = tensor([1])]; - tensor input_189_pad_0 = const()[name = tensor("input_189_pad_0"), val = tensor([0, 0])]; - tensor input_189_dilations_0 = const()[name = tensor("input_189_dilations_0"), val = tensor([1])]; - tensor input_189_groups_0 = const()[name = tensor("input_189_groups_0"), val = tensor(1)]; - tensor input_189 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_189_dilations_0, groups = input_189_groups_0, pad = input_189_pad_0, pad_type = input_189_pad_type_0, strides = input_189_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_187)[name = tensor("input_189")]; - tensor conv_out_21_perm_0 = const()[name = tensor("conv_out_21_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1408_begin_0 = const()[name = tensor("op_1408_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1408_end_0 = const()[name = tensor("op_1408_end_0"), val = tensor([1, 16, 256])]; - tensor var_1408_end_mask_0 = const()[name = tensor("op_1408_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_21 = transpose(perm = conv_out_21_perm_0, x = input_189)[name = tensor("transpose_53")]; - tensor var_1408 = slice_by_index(begin = var_1408_begin_0, end = var_1408_end_0, end_mask = var_1408_end_mask_0, x = conv_out_21)[name = tensor("op_1408")]; - tensor var_1411_begin_0 = const()[name = tensor("op_1411_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1411_end_0 = const()[name = tensor("op_1411_end_0"), val = tensor([1, 2, 256])]; - tensor var_1411_end_mask_0 = const()[name = tensor("op_1411_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1411 = slice_by_index(begin = var_1411_begin_0, end = var_1411_end_0, end_mask = var_1411_end_mask_0, x = x_31)[name = tensor("op_1411")]; - tensor var_1414_begin_0 = const()[name = tensor("op_1414_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1414_end_0 = const()[name = tensor("op_1414_end_0"), val = tensor([1, 16, 256])]; - tensor var_1414_end_mask_0 = const()[name = tensor("op_1414_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1414 = slice_by_index(begin = var_1414_begin_0, end = var_1414_end_0, end_mask = var_1414_end_mask_0, x = input_181)[name = tensor("op_1414")]; + tensor x_expanded_21 = tile(reps = x_expanded_21_reps_0, x = var_897)[name = tensor("x_expanded_21")]; + tensor var_911 = sub(x = var_18, y = fill_mask_21)[name = tensor("op_911")]; + tensor var_912 = mul(x = window_25, y = var_911)[name = tensor("op_912")]; + tensor var_913 = mul(x = x_expanded_21, y = fill_mask_21)[name = tensor("op_913")]; + tensor filled_window_21 = add(x = var_912, y = var_913)[name = tensor("filled_window_21")]; + tensor var_915 = greater_equal(x = filled_21, y = var_17)[name = tensor("op_915")]; + tensor var_916_dtype_0 = const()[name = tensor("op_916_dtype_0"), val = tensor("fp32")]; + tensor var_917 = const()[name = tensor("op_917"), val = tensor([1, 1, 1])]; + tensor var_916 = cast(dtype = var_916_dtype_0, x = var_915)[name = tensor("cast_170")]; + tensor use_roll_21 = reshape(shape = var_917, x = var_916)[name = tensor("use_roll_21")]; + tensor var_919 = sub(x = var_18, y = use_roll_21)[name = tensor("op_919")]; + tensor var_920 = mul(x = filled_window_21, y = var_919)[name = tensor("op_920")]; + tensor var_921 = mul(x = rolled_21, y = use_roll_21)[name = tensor("op_921")]; + tensor window_27 = add(x = var_920, y = var_921)[name = tensor("window_27")]; + tensor var_923 = const()[name = tensor("op_923"), val = tensor(0x1p+0)]; + tensor var_924 = add(x = filled_21, y = var_923)[name = tensor("op_924")]; + tensor clip_21 = clip(alpha = var_22, beta = var_17, x = var_924)[name = tensor("clip_21")]; + tensor var_927_begin_0 = const()[name = tensor("op_927_begin_0"), val = tensor([0, 1, 0])]; + tensor var_927_end_0 = const()[name = tensor("op_927_end_0"), val = tensor([1, 2, 256])]; + tensor var_927_end_mask_0 = const()[name = tensor("op_927_end_mask_0"), val = tensor([true, false, true])]; + tensor var_927 = slice_by_index(begin = var_927_begin_0, end = var_927_end_0, end_mask = var_927_end_mask_0, x = x_15)[name = tensor("op_927")]; + tensor var_930_begin_0 = const()[name = tensor("op_930_begin_0"), val = tensor([0, 1, 0])]; + tensor var_930_end_0 = const()[name = tensor("op_930_end_0"), val = tensor([1, 16, 256])]; + tensor var_930_end_mask_0 = const()[name = tensor("op_930_end_mask_0"), val = tensor([true, true, true])]; + tensor var_930 = slice_by_index(begin = var_930_begin_0, end = var_930_end_0, end_mask = var_930_end_mask_0, x = window_27)[name = tensor("op_930")]; tensor rolled_23_interleave_0 = const()[name = tensor("rolled_23_interleave_0"), val = tensor(false)]; - tensor rolled_23 = concat(axis = var_34, interleave = rolled_23_interleave_0, values = (var_1414, var_1411))[name = tensor("rolled_23")]; + tensor rolled_23 = concat(axis = var_34, interleave = rolled_23_interleave_0, values = (var_930, var_927))[name = tensor("rolled_23")]; tensor clip_22 = clip(alpha = var_22, beta = var_21, x = clip_21)[name = tensor("clip_22")]; - tensor var_1419 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_1419")]; - tensor var_1420_dtype_0 = const()[name = tensor("op_1420_dtype_0"), val = tensor("fp32")]; - tensor var_1421 = const()[name = tensor("op_1421"), val = tensor([1, 16, 1])]; - tensor var_1420 = cast(dtype = var_1420_dtype_0, x = var_1419)[name = tensor("cast_169")]; - tensor fill_mask_23 = reshape(shape = var_1421, x = var_1420)[name = tensor("fill_mask_23")]; + tensor var_935 = equal(x = encoder__conv_positions, y = clip_22)[name = tensor("op_935")]; + tensor var_936_dtype_0 = const()[name = tensor("op_936_dtype_0"), val = tensor("fp32")]; + tensor var_937 = const()[name = tensor("op_937"), val = tensor([1, 16, 1])]; + tensor var_936 = cast(dtype = var_936_dtype_0, x = var_935)[name = tensor("cast_169")]; + tensor fill_mask_23 = reshape(shape = var_937, x = var_936)[name = tensor("fill_mask_23")]; tensor x_expanded_23_reps_0 = const()[name = tensor("x_expanded_23_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_23 = tile(reps = x_expanded_23_reps_0, x = var_1411)[name = tensor("x_expanded_23")]; - tensor var_1425 = sub(x = var_18, y = fill_mask_23)[name = tensor("op_1425")]; - tensor var_1426 = mul(x = input_181, y = var_1425)[name = tensor("op_1426")]; - tensor var_1427 = mul(x = x_expanded_23, y = fill_mask_23)[name = tensor("op_1427")]; - tensor filled_window_23 = add(x = var_1426, y = var_1427)[name = tensor("filled_window_23")]; - tensor var_1429 = greater_equal(x = clip_21, y = var_17)[name = tensor("op_1429")]; - tensor var_1430_dtype_0 = const()[name = tensor("op_1430_dtype_0"), val = tensor("fp32")]; - tensor var_1431 = const()[name = tensor("op_1431"), val = tensor([1, 1, 1])]; - tensor var_1430 = cast(dtype = var_1430_dtype_0, x = var_1429)[name = tensor("cast_168")]; - tensor use_roll_23 = reshape(shape = var_1431, x = var_1430)[name = tensor("use_roll_23")]; - tensor var_1433 = sub(x = var_18, y = use_roll_23)[name = tensor("op_1433")]; - tensor var_1434 = mul(x = filled_window_23, y = var_1433)[name = tensor("op_1434")]; - tensor var_1435 = mul(x = rolled_23, y = use_roll_23)[name = tensor("op_1435")]; - tensor input_191 = add(x = var_1434, y = var_1435)[name = tensor("input_191")]; - tensor var_1437 = const()[name = tensor("op_1437"), val = tensor(0x1p+0)]; - tensor var_1438 = add(x = clip_21, y = var_1437)[name = tensor("op_1438")]; - tensor clip_23 = clip(alpha = var_22, beta = var_17, x = var_1438)[name = tensor("clip_23")]; - tensor x_35_axes_0 = const()[name = tensor("x_35_axes_0"), val = tensor([-1])]; - tensor x_35 = layer_norm(axes = x_35_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_191)[name = tensor("x_35")]; - tensor input_193_perm_0 = const()[name = tensor("input_193_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_77_pad_type_0 = const()[name = tensor("inputs_77_pad_type_0"), val = tensor("valid")]; - tensor inputs_77_strides_0 = const()[name = tensor("inputs_77_strides_0"), val = tensor([1])]; - tensor inputs_77_pad_0 = const()[name = tensor("inputs_77_pad_0"), val = tensor([0, 0])]; - tensor inputs_77_dilations_0 = const()[name = tensor("inputs_77_dilations_0"), val = tensor([1])]; - tensor inputs_77_groups_0 = const()[name = tensor("inputs_77_groups_0"), val = tensor(1)]; - tensor input_193 = transpose(perm = input_193_perm_0, x = x_35)[name = tensor("transpose_52")]; - tensor inputs_77 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_77_dilations_0, groups = inputs_77_groups_0, pad = inputs_77_pad_0, pad_type = inputs_77_pad_type_0, strides = inputs_77_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_193)[name = tensor("inputs_77")]; - tensor var_1459_split_sizes_0 = const()[name = tensor("op_1459_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1459_axis_0 = const()[name = tensor("op_1459_axis_0"), val = tensor(1)]; - tensor var_1459_0, tensor var_1459_1 = split(axis = var_1459_axis_0, split_sizes = var_1459_split_sizes_0, x = inputs_77)[name = tensor("op_1459")]; - tensor var_1461 = sigmoid(x = var_1459_1)[name = tensor("op_1461")]; - tensor inputs_79 = mul(x = var_1459_0, y = var_1461)[name = tensor("inputs_79")]; - tensor outputs_aug_23_pad_type_0 = const()[name = tensor("outputs_aug_23_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_23_pad_0 = const()[name = tensor("outputs_aug_23_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_23_groups_0 = const()[name = tensor("outputs_aug_23_groups_0"), val = tensor(256)]; - tensor outputs_aug_23_strides_0 = const()[name = tensor("outputs_aug_23_strides_0"), val = tensor([1])]; - tensor outputs_aug_23_dilations_0 = const()[name = tensor("outputs_aug_23_dilations_0"), val = tensor([1])]; - tensor outputs_aug_23 = conv(dilations = outputs_aug_23_dilations_0, groups = outputs_aug_23_groups_0, pad = outputs_aug_23_pad_0, pad_type = outputs_aug_23_pad_type_0, strides = outputs_aug_23_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_79)[name = tensor("outputs_aug_23")]; - tensor input_195_begin_0 = const()[name = tensor("input_195_begin_0"), val = tensor([0, 0, 0])]; - tensor input_195_end_0 = const()[name = tensor("input_195_end_0"), val = tensor([1, 256, 16])]; - tensor input_195_end_mask_0 = const()[name = tensor("input_195_end_mask_0"), val = tensor([true, true, false])]; - tensor input_195 = slice_by_index(begin = input_195_begin_0, end = input_195_end_0, end_mask = input_195_end_mask_0, x = outputs_aug_23)[name = tensor("input_195")]; - tensor inputs_81 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_195)[name = tensor("inputs_81")]; - tensor input_197 = silu(x = inputs_81)[name = tensor("input_197")]; - tensor input_199_pad_type_0 = const()[name = tensor("input_199_pad_type_0"), val = tensor("valid")]; - tensor input_199_strides_0 = const()[name = tensor("input_199_strides_0"), val = tensor([1])]; - tensor input_199_pad_0 = const()[name = tensor("input_199_pad_0"), val = tensor([0, 0])]; - tensor input_199_dilations_0 = const()[name = tensor("input_199_dilations_0"), val = tensor([1])]; - tensor input_199_groups_0 = const()[name = tensor("input_199_groups_0"), val = tensor(1)]; - tensor input_199 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_199_dilations_0, groups = input_199_groups_0, pad = input_199_pad_0, pad_type = input_199_pad_type_0, strides = input_199_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_197)[name = tensor("input_199")]; - tensor conv_out_23_perm_0 = const()[name = tensor("conv_out_23_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1492_begin_0 = const()[name = tensor("op_1492_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1492_end_0 = const()[name = tensor("op_1492_end_0"), val = tensor([1, 16, 256])]; - tensor var_1492_end_mask_0 = const()[name = tensor("op_1492_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_23 = transpose(perm = conv_out_23_perm_0, x = input_199)[name = tensor("transpose_51")]; - tensor var_1492 = slice_by_index(begin = var_1492_begin_0, end = var_1492_end_0, end_mask = var_1492_end_mask_0, x = conv_out_23)[name = tensor("op_1492")]; - tensor var_1495_begin_0 = const()[name = tensor("op_1495_begin_0"), val = tensor([0, 2, 0])]; - tensor var_1495_end_0 = const()[name = tensor("op_1495_end_0"), val = tensor([1, 3, 256])]; - tensor var_1495_end_mask_0 = const()[name = tensor("op_1495_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1495 = slice_by_index(begin = var_1495_begin_0, end = var_1495_end_0, end_mask = var_1495_end_mask_0, x = x_31)[name = tensor("op_1495")]; - tensor var_1498_begin_0 = const()[name = tensor("op_1498_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1498_end_0 = const()[name = tensor("op_1498_end_0"), val = tensor([1, 16, 256])]; - tensor var_1498_end_mask_0 = const()[name = tensor("op_1498_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1498 = slice_by_index(begin = var_1498_begin_0, end = var_1498_end_0, end_mask = var_1498_end_mask_0, x = input_191)[name = tensor("op_1498")]; + tensor x_expanded_23 = tile(reps = x_expanded_23_reps_0, x = var_927)[name = tensor("x_expanded_23")]; + tensor var_941 = sub(x = var_18, y = fill_mask_23)[name = tensor("op_941")]; + tensor var_942 = mul(x = window_27, y = var_941)[name = tensor("op_942")]; + tensor var_943 = mul(x = x_expanded_23, y = fill_mask_23)[name = tensor("op_943")]; + tensor filled_window_23 = add(x = var_942, y = var_943)[name = tensor("filled_window_23")]; + tensor var_945 = greater_equal(x = clip_21, y = var_17)[name = tensor("op_945")]; + tensor var_946_dtype_0 = const()[name = tensor("op_946_dtype_0"), val = tensor("fp32")]; + tensor var_947 = const()[name = tensor("op_947"), val = tensor([1, 1, 1])]; + tensor var_946 = cast(dtype = var_946_dtype_0, x = var_945)[name = tensor("cast_168")]; + tensor use_roll_23 = reshape(shape = var_947, x = var_946)[name = tensor("use_roll_23")]; + tensor var_949 = sub(x = var_18, y = use_roll_23)[name = tensor("op_949")]; + tensor var_950 = mul(x = filled_window_23, y = var_949)[name = tensor("op_950")]; + tensor var_951 = mul(x = rolled_23, y = use_roll_23)[name = tensor("op_951")]; + tensor window_29 = add(x = var_950, y = var_951)[name = tensor("window_29")]; + tensor var_953 = const()[name = tensor("op_953"), val = tensor(0x1p+0)]; + tensor var_954 = add(x = clip_21, y = var_953)[name = tensor("op_954")]; + tensor clip_23 = clip(alpha = var_22, beta = var_17, x = var_954)[name = tensor("clip_23")]; + tensor var_957_begin_0 = const()[name = tensor("op_957_begin_0"), val = tensor([0, 2, 0])]; + tensor var_957_end_0 = const()[name = tensor("op_957_end_0"), val = tensor([1, 3, 256])]; + tensor var_957_end_mask_0 = const()[name = tensor("op_957_end_mask_0"), val = tensor([true, false, true])]; + tensor var_957 = slice_by_index(begin = var_957_begin_0, end = var_957_end_0, end_mask = var_957_end_mask_0, x = x_15)[name = tensor("op_957")]; + tensor var_960_begin_0 = const()[name = tensor("op_960_begin_0"), val = tensor([0, 1, 0])]; + tensor var_960_end_0 = const()[name = tensor("op_960_end_0"), val = tensor([1, 16, 256])]; + tensor var_960_end_mask_0 = const()[name = tensor("op_960_end_mask_0"), val = tensor([true, true, true])]; + tensor var_960 = slice_by_index(begin = var_960_begin_0, end = var_960_end_0, end_mask = var_960_end_mask_0, x = window_29)[name = tensor("op_960")]; tensor rolled_25_interleave_0 = const()[name = tensor("rolled_25_interleave_0"), val = tensor(false)]; - tensor rolled_25 = concat(axis = var_34, interleave = rolled_25_interleave_0, values = (var_1498, var_1495))[name = tensor("rolled_25")]; + tensor rolled_25 = concat(axis = var_34, interleave = rolled_25_interleave_0, values = (var_960, var_957))[name = tensor("rolled_25")]; tensor clip_24 = clip(alpha = var_22, beta = var_21, x = clip_23)[name = tensor("clip_24")]; - tensor var_1503 = equal(x = encoder__conv_positions, y = clip_24)[name = tensor("op_1503")]; - tensor var_1504_dtype_0 = const()[name = tensor("op_1504_dtype_0"), val = tensor("fp32")]; - tensor var_1505 = const()[name = tensor("op_1505"), val = tensor([1, 16, 1])]; - tensor var_1504 = cast(dtype = var_1504_dtype_0, x = var_1503)[name = tensor("cast_167")]; - tensor fill_mask_25 = reshape(shape = var_1505, x = var_1504)[name = tensor("fill_mask_25")]; + tensor var_965 = equal(x = encoder__conv_positions, y = clip_24)[name = tensor("op_965")]; + tensor var_966_dtype_0 = const()[name = tensor("op_966_dtype_0"), val = tensor("fp32")]; + tensor var_967 = const()[name = tensor("op_967"), val = tensor([1, 16, 1])]; + tensor var_966 = cast(dtype = var_966_dtype_0, x = var_965)[name = tensor("cast_167")]; + tensor fill_mask_25 = reshape(shape = var_967, x = var_966)[name = tensor("fill_mask_25")]; tensor x_expanded_25_reps_0 = const()[name = tensor("x_expanded_25_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_25 = tile(reps = x_expanded_25_reps_0, x = var_1495)[name = tensor("x_expanded_25")]; - tensor var_1509 = sub(x = var_18, y = fill_mask_25)[name = tensor("op_1509")]; - tensor var_1510 = mul(x = input_191, y = var_1509)[name = tensor("op_1510")]; - tensor var_1511 = mul(x = x_expanded_25, y = fill_mask_25)[name = tensor("op_1511")]; - tensor filled_window_25 = add(x = var_1510, y = var_1511)[name = tensor("filled_window_25")]; - tensor var_1513 = greater_equal(x = clip_23, y = var_17)[name = tensor("op_1513")]; - tensor var_1514_dtype_0 = const()[name = tensor("op_1514_dtype_0"), val = tensor("fp32")]; - tensor var_1515 = const()[name = tensor("op_1515"), val = tensor([1, 1, 1])]; - tensor var_1514 = cast(dtype = var_1514_dtype_0, x = var_1513)[name = tensor("cast_166")]; - tensor use_roll_25 = reshape(shape = var_1515, x = var_1514)[name = tensor("use_roll_25")]; - tensor var_1517 = sub(x = var_18, y = use_roll_25)[name = tensor("op_1517")]; - tensor var_1518 = mul(x = filled_window_25, y = var_1517)[name = tensor("op_1518")]; - tensor var_1519 = mul(x = rolled_25, y = use_roll_25)[name = tensor("op_1519")]; - tensor input_201 = add(x = var_1518, y = var_1519)[name = tensor("input_201")]; - tensor var_1521 = const()[name = tensor("op_1521"), val = tensor(0x1p+0)]; - tensor var_1522 = add(x = clip_23, y = var_1521)[name = tensor("op_1522")]; - tensor clip_25 = clip(alpha = var_22, beta = var_17, x = var_1522)[name = tensor("clip_25")]; - tensor x_37_axes_0 = const()[name = tensor("x_37_axes_0"), val = tensor([-1])]; - tensor x_37 = layer_norm(axes = x_37_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_201)[name = tensor("x_37")]; - tensor input_203_perm_0 = const()[name = tensor("input_203_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_83_pad_type_0 = const()[name = tensor("inputs_83_pad_type_0"), val = tensor("valid")]; - tensor inputs_83_strides_0 = const()[name = tensor("inputs_83_strides_0"), val = tensor([1])]; - tensor inputs_83_pad_0 = const()[name = tensor("inputs_83_pad_0"), val = tensor([0, 0])]; - tensor inputs_83_dilations_0 = const()[name = tensor("inputs_83_dilations_0"), val = tensor([1])]; - tensor inputs_83_groups_0 = const()[name = tensor("inputs_83_groups_0"), val = tensor(1)]; - tensor input_203 = transpose(perm = input_203_perm_0, x = x_37)[name = tensor("transpose_50")]; - tensor inputs_83 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_83_dilations_0, groups = inputs_83_groups_0, pad = inputs_83_pad_0, pad_type = inputs_83_pad_type_0, strides = inputs_83_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_203)[name = tensor("inputs_83")]; - tensor var_1543_split_sizes_0 = const()[name = tensor("op_1543_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1543_axis_0 = const()[name = tensor("op_1543_axis_0"), val = tensor(1)]; - tensor var_1543_0, tensor var_1543_1 = split(axis = var_1543_axis_0, split_sizes = var_1543_split_sizes_0, x = inputs_83)[name = tensor("op_1543")]; - tensor var_1545 = sigmoid(x = var_1543_1)[name = tensor("op_1545")]; - tensor inputs_85 = mul(x = var_1543_0, y = var_1545)[name = tensor("inputs_85")]; - tensor outputs_aug_25_pad_type_0 = const()[name = tensor("outputs_aug_25_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_25_pad_0 = const()[name = tensor("outputs_aug_25_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_25_groups_0 = const()[name = tensor("outputs_aug_25_groups_0"), val = tensor(256)]; - tensor outputs_aug_25_strides_0 = const()[name = tensor("outputs_aug_25_strides_0"), val = tensor([1])]; - tensor outputs_aug_25_dilations_0 = const()[name = tensor("outputs_aug_25_dilations_0"), val = tensor([1])]; - tensor outputs_aug_25 = conv(dilations = outputs_aug_25_dilations_0, groups = outputs_aug_25_groups_0, pad = outputs_aug_25_pad_0, pad_type = outputs_aug_25_pad_type_0, strides = outputs_aug_25_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_85)[name = tensor("outputs_aug_25")]; - tensor input_205_begin_0 = const()[name = tensor("input_205_begin_0"), val = tensor([0, 0, 0])]; - tensor input_205_end_0 = const()[name = tensor("input_205_end_0"), val = tensor([1, 256, 16])]; - tensor input_205_end_mask_0 = const()[name = tensor("input_205_end_mask_0"), val = tensor([true, true, false])]; - tensor input_205 = slice_by_index(begin = input_205_begin_0, end = input_205_end_0, end_mask = input_205_end_mask_0, x = outputs_aug_25)[name = tensor("input_205")]; - tensor inputs_87 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_205)[name = tensor("inputs_87")]; - tensor input_207 = silu(x = inputs_87)[name = tensor("input_207")]; - tensor input_209_pad_type_0 = const()[name = tensor("input_209_pad_type_0"), val = tensor("valid")]; - tensor input_209_strides_0 = const()[name = tensor("input_209_strides_0"), val = tensor([1])]; - tensor input_209_pad_0 = const()[name = tensor("input_209_pad_0"), val = tensor([0, 0])]; - tensor input_209_dilations_0 = const()[name = tensor("input_209_dilations_0"), val = tensor([1])]; - tensor input_209_groups_0 = const()[name = tensor("input_209_groups_0"), val = tensor(1)]; - tensor input_209 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_209_dilations_0, groups = input_209_groups_0, pad = input_209_pad_0, pad_type = input_209_pad_type_0, strides = input_209_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_207)[name = tensor("input_209")]; - tensor conv_out_25_perm_0 = const()[name = tensor("conv_out_25_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1576_begin_0 = const()[name = tensor("op_1576_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1576_end_0 = const()[name = tensor("op_1576_end_0"), val = tensor([1, 16, 256])]; - tensor var_1576_end_mask_0 = const()[name = tensor("op_1576_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_25 = transpose(perm = conv_out_25_perm_0, x = input_209)[name = tensor("transpose_49")]; - tensor var_1576 = slice_by_index(begin = var_1576_begin_0, end = var_1576_end_0, end_mask = var_1576_end_mask_0, x = conv_out_25)[name = tensor("op_1576")]; - tensor var_1579_begin_0 = const()[name = tensor("op_1579_begin_0"), val = tensor([0, 3, 0])]; - tensor var_1579_end_0 = const()[name = tensor("op_1579_end_0"), val = tensor([1, 4, 256])]; - tensor var_1579_end_mask_0 = const()[name = tensor("op_1579_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1579 = slice_by_index(begin = var_1579_begin_0, end = var_1579_end_0, end_mask = var_1579_end_mask_0, x = x_31)[name = tensor("op_1579")]; - tensor var_1582_begin_0 = const()[name = tensor("op_1582_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1582_end_0 = const()[name = tensor("op_1582_end_0"), val = tensor([1, 16, 256])]; - tensor var_1582_end_mask_0 = const()[name = tensor("op_1582_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1582 = slice_by_index(begin = var_1582_begin_0, end = var_1582_end_0, end_mask = var_1582_end_mask_0, x = input_201)[name = tensor("op_1582")]; + tensor x_expanded_25 = tile(reps = x_expanded_25_reps_0, x = var_957)[name = tensor("x_expanded_25")]; + tensor var_971 = sub(x = var_18, y = fill_mask_25)[name = tensor("op_971")]; + tensor var_972 = mul(x = window_29, y = var_971)[name = tensor("op_972")]; + tensor var_973 = mul(x = x_expanded_25, y = fill_mask_25)[name = tensor("op_973")]; + tensor filled_window_25 = add(x = var_972, y = var_973)[name = tensor("filled_window_25")]; + tensor var_975 = greater_equal(x = clip_23, y = var_17)[name = tensor("op_975")]; + tensor var_976_dtype_0 = const()[name = tensor("op_976_dtype_0"), val = tensor("fp32")]; + tensor var_977 = const()[name = tensor("op_977"), val = tensor([1, 1, 1])]; + tensor var_976 = cast(dtype = var_976_dtype_0, x = var_975)[name = tensor("cast_166")]; + tensor use_roll_25 = reshape(shape = var_977, x = var_976)[name = tensor("use_roll_25")]; + tensor var_979 = sub(x = var_18, y = use_roll_25)[name = tensor("op_979")]; + tensor var_980 = mul(x = filled_window_25, y = var_979)[name = tensor("op_980")]; + tensor var_981 = mul(x = rolled_25, y = use_roll_25)[name = tensor("op_981")]; + tensor window_31 = add(x = var_980, y = var_981)[name = tensor("window_31")]; + tensor var_983 = const()[name = tensor("op_983"), val = tensor(0x1p+0)]; + tensor var_984 = add(x = clip_23, y = var_983)[name = tensor("op_984")]; + tensor clip_25 = clip(alpha = var_22, beta = var_17, x = var_984)[name = tensor("clip_25")]; + tensor var_987_begin_0 = const()[name = tensor("op_987_begin_0"), val = tensor([0, 3, 0])]; + tensor var_987_end_0 = const()[name = tensor("op_987_end_0"), val = tensor([1, 4, 256])]; + tensor var_987_end_mask_0 = const()[name = tensor("op_987_end_mask_0"), val = tensor([true, false, true])]; + tensor var_987 = slice_by_index(begin = var_987_begin_0, end = var_987_end_0, end_mask = var_987_end_mask_0, x = x_15)[name = tensor("op_987")]; + tensor var_990_begin_0 = const()[name = tensor("op_990_begin_0"), val = tensor([0, 1, 0])]; + tensor var_990_end_0 = const()[name = tensor("op_990_end_0"), val = tensor([1, 16, 256])]; + tensor var_990_end_mask_0 = const()[name = tensor("op_990_end_mask_0"), val = tensor([true, true, true])]; + tensor var_990 = slice_by_index(begin = var_990_begin_0, end = var_990_end_0, end_mask = var_990_end_mask_0, x = window_31)[name = tensor("op_990")]; tensor rolled_27_interleave_0 = const()[name = tensor("rolled_27_interleave_0"), val = tensor(false)]; - tensor rolled_27 = concat(axis = var_34, interleave = rolled_27_interleave_0, values = (var_1582, var_1579))[name = tensor("rolled_27")]; + tensor rolled_27 = concat(axis = var_34, interleave = rolled_27_interleave_0, values = (var_990, var_987))[name = tensor("rolled_27")]; tensor clip_26 = clip(alpha = var_22, beta = var_21, x = clip_25)[name = tensor("clip_26")]; - tensor var_1587 = equal(x = encoder__conv_positions, y = clip_26)[name = tensor("op_1587")]; - tensor var_1588_dtype_0 = const()[name = tensor("op_1588_dtype_0"), val = tensor("fp32")]; - tensor var_1589 = const()[name = tensor("op_1589"), val = tensor([1, 16, 1])]; - tensor var_1588 = cast(dtype = var_1588_dtype_0, x = var_1587)[name = tensor("cast_165")]; - tensor fill_mask_27 = reshape(shape = var_1589, x = var_1588)[name = tensor("fill_mask_27")]; + tensor var_995 = equal(x = encoder__conv_positions, y = clip_26)[name = tensor("op_995")]; + tensor var_996_dtype_0 = const()[name = tensor("op_996_dtype_0"), val = tensor("fp32")]; + tensor var_997 = const()[name = tensor("op_997"), val = tensor([1, 16, 1])]; + tensor var_996 = cast(dtype = var_996_dtype_0, x = var_995)[name = tensor("cast_165")]; + tensor fill_mask_27 = reshape(shape = var_997, x = var_996)[name = tensor("fill_mask_27")]; tensor x_expanded_27_reps_0 = const()[name = tensor("x_expanded_27_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_27 = tile(reps = x_expanded_27_reps_0, x = var_1579)[name = tensor("x_expanded_27")]; - tensor var_1593 = sub(x = var_18, y = fill_mask_27)[name = tensor("op_1593")]; - tensor var_1594 = mul(x = input_201, y = var_1593)[name = tensor("op_1594")]; - tensor var_1595 = mul(x = x_expanded_27, y = fill_mask_27)[name = tensor("op_1595")]; - tensor filled_window_27 = add(x = var_1594, y = var_1595)[name = tensor("filled_window_27")]; - tensor var_1597 = greater_equal(x = clip_25, y = var_17)[name = tensor("op_1597")]; - tensor var_1598_dtype_0 = const()[name = tensor("op_1598_dtype_0"), val = tensor("fp32")]; - tensor var_1599 = const()[name = tensor("op_1599"), val = tensor([1, 1, 1])]; - tensor var_1598 = cast(dtype = var_1598_dtype_0, x = var_1597)[name = tensor("cast_164")]; - tensor use_roll_27 = reshape(shape = var_1599, x = var_1598)[name = tensor("use_roll_27")]; - tensor var_1601 = sub(x = var_18, y = use_roll_27)[name = tensor("op_1601")]; - tensor var_1602 = mul(x = filled_window_27, y = var_1601)[name = tensor("op_1602")]; - tensor var_1603 = mul(x = rolled_27, y = use_roll_27)[name = tensor("op_1603")]; - tensor input_211 = add(x = var_1602, y = var_1603)[name = tensor("input_211")]; - tensor var_1605 = const()[name = tensor("op_1605"), val = tensor(0x1p+0)]; - tensor var_1606 = add(x = clip_25, y = var_1605)[name = tensor("op_1606")]; - tensor clip_27 = clip(alpha = var_22, beta = var_17, x = var_1606)[name = tensor("clip_27")]; - tensor x_39_axes_0 = const()[name = tensor("x_39_axes_0"), val = tensor([-1])]; - tensor x_39 = layer_norm(axes = x_39_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_211)[name = tensor("x_39")]; - tensor input_213_perm_0 = const()[name = tensor("input_213_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_89_pad_type_0 = const()[name = tensor("inputs_89_pad_type_0"), val = tensor("valid")]; - tensor inputs_89_strides_0 = const()[name = tensor("inputs_89_strides_0"), val = tensor([1])]; - tensor inputs_89_pad_0 = const()[name = tensor("inputs_89_pad_0"), val = tensor([0, 0])]; - tensor inputs_89_dilations_0 = const()[name = tensor("inputs_89_dilations_0"), val = tensor([1])]; - tensor inputs_89_groups_0 = const()[name = tensor("inputs_89_groups_0"), val = tensor(1)]; - tensor input_213 = transpose(perm = input_213_perm_0, x = x_39)[name = tensor("transpose_48")]; - tensor inputs_89 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_89_dilations_0, groups = inputs_89_groups_0, pad = inputs_89_pad_0, pad_type = inputs_89_pad_type_0, strides = inputs_89_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_213)[name = tensor("inputs_89")]; - tensor var_1627_split_sizes_0 = const()[name = tensor("op_1627_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1627_axis_0 = const()[name = tensor("op_1627_axis_0"), val = tensor(1)]; - tensor var_1627_0, tensor var_1627_1 = split(axis = var_1627_axis_0, split_sizes = var_1627_split_sizes_0, x = inputs_89)[name = tensor("op_1627")]; - tensor var_1629 = sigmoid(x = var_1627_1)[name = tensor("op_1629")]; - tensor inputs_91 = mul(x = var_1627_0, y = var_1629)[name = tensor("inputs_91")]; - tensor outputs_aug_27_pad_type_0 = const()[name = tensor("outputs_aug_27_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_27_pad_0 = const()[name = tensor("outputs_aug_27_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_27_groups_0 = const()[name = tensor("outputs_aug_27_groups_0"), val = tensor(256)]; - tensor outputs_aug_27_strides_0 = const()[name = tensor("outputs_aug_27_strides_0"), val = tensor([1])]; - tensor outputs_aug_27_dilations_0 = const()[name = tensor("outputs_aug_27_dilations_0"), val = tensor([1])]; - tensor outputs_aug_27 = conv(dilations = outputs_aug_27_dilations_0, groups = outputs_aug_27_groups_0, pad = outputs_aug_27_pad_0, pad_type = outputs_aug_27_pad_type_0, strides = outputs_aug_27_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_91)[name = tensor("outputs_aug_27")]; - tensor input_215_begin_0 = const()[name = tensor("input_215_begin_0"), val = tensor([0, 0, 0])]; - tensor input_215_end_0 = const()[name = tensor("input_215_end_0"), val = tensor([1, 256, 16])]; - tensor input_215_end_mask_0 = const()[name = tensor("input_215_end_mask_0"), val = tensor([true, true, false])]; - tensor input_215 = slice_by_index(begin = input_215_begin_0, end = input_215_end_0, end_mask = input_215_end_mask_0, x = outputs_aug_27)[name = tensor("input_215")]; - tensor inputs_93 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_215)[name = tensor("inputs_93")]; - tensor input_217 = silu(x = inputs_93)[name = tensor("input_217")]; - tensor input_219_pad_type_0 = const()[name = tensor("input_219_pad_type_0"), val = tensor("valid")]; - tensor input_219_strides_0 = const()[name = tensor("input_219_strides_0"), val = tensor([1])]; - tensor input_219_pad_0 = const()[name = tensor("input_219_pad_0"), val = tensor([0, 0])]; - tensor input_219_dilations_0 = const()[name = tensor("input_219_dilations_0"), val = tensor([1])]; - tensor input_219_groups_0 = const()[name = tensor("input_219_groups_0"), val = tensor(1)]; - tensor input_219 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_219_dilations_0, groups = input_219_groups_0, pad = input_219_pad_0, pad_type = input_219_pad_type_0, strides = input_219_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_217)[name = tensor("input_219")]; - tensor conv_out_27_perm_0 = const()[name = tensor("conv_out_27_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1660_begin_0 = const()[name = tensor("op_1660_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1660_end_0 = const()[name = tensor("op_1660_end_0"), val = tensor([1, 16, 256])]; - tensor var_1660_end_mask_0 = const()[name = tensor("op_1660_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_27 = transpose(perm = conv_out_27_perm_0, x = input_219)[name = tensor("transpose_47")]; - tensor var_1660 = slice_by_index(begin = var_1660_begin_0, end = var_1660_end_0, end_mask = var_1660_end_mask_0, x = conv_out_27)[name = tensor("op_1660")]; - tensor var_1663_begin_0 = const()[name = tensor("op_1663_begin_0"), val = tensor([0, 4, 0])]; - tensor var_1663_end_0 = const()[name = tensor("op_1663_end_0"), val = tensor([1, 1, 256])]; - tensor var_1663_end_mask_0 = const()[name = tensor("op_1663_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1663 = slice_by_index(begin = var_1663_begin_0, end = var_1663_end_0, end_mask = var_1663_end_mask_0, x = x_31)[name = tensor("op_1663")]; - tensor var_1666_begin_0 = const()[name = tensor("op_1666_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1666_end_0 = const()[name = tensor("op_1666_end_0"), val = tensor([1, 16, 256])]; - tensor var_1666_end_mask_0 = const()[name = tensor("op_1666_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1666 = slice_by_index(begin = var_1666_begin_0, end = var_1666_end_0, end_mask = var_1666_end_mask_0, x = input_211)[name = tensor("op_1666")]; + tensor x_expanded_27 = tile(reps = x_expanded_27_reps_0, x = var_987)[name = tensor("x_expanded_27")]; + tensor var_1001 = sub(x = var_18, y = fill_mask_27)[name = tensor("op_1001")]; + tensor var_1002 = mul(x = window_31, y = var_1001)[name = tensor("op_1002")]; + tensor var_1003 = mul(x = x_expanded_27, y = fill_mask_27)[name = tensor("op_1003")]; + tensor filled_window_27 = add(x = var_1002, y = var_1003)[name = tensor("filled_window_27")]; + tensor var_1005 = greater_equal(x = clip_25, y = var_17)[name = tensor("op_1005")]; + tensor var_1006_dtype_0 = const()[name = tensor("op_1006_dtype_0"), val = tensor("fp32")]; + tensor var_1007 = const()[name = tensor("op_1007"), val = tensor([1, 1, 1])]; + tensor var_1006 = cast(dtype = var_1006_dtype_0, x = var_1005)[name = tensor("cast_164")]; + tensor use_roll_27 = reshape(shape = var_1007, x = var_1006)[name = tensor("use_roll_27")]; + tensor var_1009 = sub(x = var_18, y = use_roll_27)[name = tensor("op_1009")]; + tensor var_1010 = mul(x = filled_window_27, y = var_1009)[name = tensor("op_1010")]; + tensor var_1011 = mul(x = rolled_27, y = use_roll_27)[name = tensor("op_1011")]; + tensor window_33 = add(x = var_1010, y = var_1011)[name = tensor("window_33")]; + tensor var_1013 = const()[name = tensor("op_1013"), val = tensor(0x1p+0)]; + tensor var_1014 = add(x = clip_25, y = var_1013)[name = tensor("op_1014")]; + tensor clip_27 = clip(alpha = var_22, beta = var_17, x = var_1014)[name = tensor("clip_27")]; + tensor var_1017_begin_0 = const()[name = tensor("op_1017_begin_0"), val = tensor([0, 4, 0])]; + tensor var_1017_end_0 = const()[name = tensor("op_1017_end_0"), val = tensor([1, 1, 256])]; + tensor var_1017_end_mask_0 = const()[name = tensor("op_1017_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1017 = slice_by_index(begin = var_1017_begin_0, end = var_1017_end_0, end_mask = var_1017_end_mask_0, x = x_15)[name = tensor("op_1017")]; + tensor var_1020_begin_0 = const()[name = tensor("op_1020_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1020_end_0 = const()[name = tensor("op_1020_end_0"), val = tensor([1, 16, 256])]; + tensor var_1020_end_mask_0 = const()[name = tensor("op_1020_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1020 = slice_by_index(begin = var_1020_begin_0, end = var_1020_end_0, end_mask = var_1020_end_mask_0, x = window_33)[name = tensor("op_1020")]; tensor rolled_29_interleave_0 = const()[name = tensor("rolled_29_interleave_0"), val = tensor(false)]; - tensor rolled_29 = concat(axis = var_34, interleave = rolled_29_interleave_0, values = (var_1666, var_1663))[name = tensor("rolled_29")]; + tensor rolled_29 = concat(axis = var_34, interleave = rolled_29_interleave_0, values = (var_1020, var_1017))[name = tensor("rolled_29")]; tensor clip_28 = clip(alpha = var_22, beta = var_21, x = clip_27)[name = tensor("clip_28")]; - tensor var_1671 = equal(x = encoder__conv_positions, y = clip_28)[name = tensor("op_1671")]; - tensor var_1672_dtype_0 = const()[name = tensor("op_1672_dtype_0"), val = tensor("fp32")]; - tensor var_1673 = const()[name = tensor("op_1673"), val = tensor([1, 16, 1])]; - tensor var_1672 = cast(dtype = var_1672_dtype_0, x = var_1671)[name = tensor("cast_163")]; - tensor fill_mask_29 = reshape(shape = var_1673, x = var_1672)[name = tensor("fill_mask_29")]; + tensor var_1025 = equal(x = encoder__conv_positions, y = clip_28)[name = tensor("op_1025")]; + tensor var_1026_dtype_0 = const()[name = tensor("op_1026_dtype_0"), val = tensor("fp32")]; + tensor var_1027 = const()[name = tensor("op_1027"), val = tensor([1, 16, 1])]; + tensor var_1026 = cast(dtype = var_1026_dtype_0, x = var_1025)[name = tensor("cast_163")]; + tensor fill_mask_29 = reshape(shape = var_1027, x = var_1026)[name = tensor("fill_mask_29")]; tensor x_expanded_29_reps_0 = const()[name = tensor("x_expanded_29_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_29 = tile(reps = x_expanded_29_reps_0, x = var_1663)[name = tensor("x_expanded_29")]; - tensor var_1677 = sub(x = var_18, y = fill_mask_29)[name = tensor("op_1677")]; - tensor var_1678 = mul(x = input_211, y = var_1677)[name = tensor("op_1678")]; - tensor var_1679 = mul(x = x_expanded_29, y = fill_mask_29)[name = tensor("op_1679")]; - tensor filled_window_29 = add(x = var_1678, y = var_1679)[name = tensor("filled_window_29")]; - tensor var_1681 = greater_equal(x = clip_27, y = var_17)[name = tensor("op_1681")]; - tensor var_1682_dtype_0 = const()[name = tensor("op_1682_dtype_0"), val = tensor("fp32")]; - tensor var_1683 = const()[name = tensor("op_1683"), val = tensor([1, 1, 1])]; - tensor var_1682 = cast(dtype = var_1682_dtype_0, x = var_1681)[name = tensor("cast_162")]; - tensor use_roll_29 = reshape(shape = var_1683, x = var_1682)[name = tensor("use_roll_29")]; - tensor var_1685 = sub(x = var_18, y = use_roll_29)[name = tensor("op_1685")]; - tensor var_1686 = mul(x = filled_window_29, y = var_1685)[name = tensor("op_1686")]; - tensor var_1687 = mul(x = rolled_29, y = use_roll_29)[name = tensor("op_1687")]; - tensor input_221 = add(x = var_1686, y = var_1687)[name = tensor("input_221")]; - tensor var_1689 = const()[name = tensor("op_1689"), val = tensor(0x1p+0)]; - tensor var_1690 = add(x = clip_27, y = var_1689)[name = tensor("op_1690")]; - tensor clip_29 = clip(alpha = var_22, beta = var_17, x = var_1690)[name = tensor("clip_29")]; - tensor x_41_axes_0 = const()[name = tensor("x_41_axes_0"), val = tensor([-1])]; - tensor x_41 = layer_norm(axes = x_41_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_221)[name = tensor("x_41")]; - tensor input_223_perm_0 = const()[name = tensor("input_223_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_95_pad_type_0 = const()[name = tensor("inputs_95_pad_type_0"), val = tensor("valid")]; - tensor inputs_95_strides_0 = const()[name = tensor("inputs_95_strides_0"), val = tensor([1])]; - tensor inputs_95_pad_0 = const()[name = tensor("inputs_95_pad_0"), val = tensor([0, 0])]; - tensor inputs_95_dilations_0 = const()[name = tensor("inputs_95_dilations_0"), val = tensor([1])]; - tensor inputs_95_groups_0 = const()[name = tensor("inputs_95_groups_0"), val = tensor(1)]; - tensor input_223 = transpose(perm = input_223_perm_0, x = x_41)[name = tensor("transpose_46")]; - tensor inputs_95 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_95_dilations_0, groups = inputs_95_groups_0, pad = inputs_95_pad_0, pad_type = inputs_95_pad_type_0, strides = inputs_95_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_223)[name = tensor("inputs_95")]; - tensor var_1711_split_sizes_0 = const()[name = tensor("op_1711_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1711_axis_0 = const()[name = tensor("op_1711_axis_0"), val = tensor(1)]; - tensor var_1711_0, tensor var_1711_1 = split(axis = var_1711_axis_0, split_sizes = var_1711_split_sizes_0, x = inputs_95)[name = tensor("op_1711")]; - tensor var_1713 = sigmoid(x = var_1711_1)[name = tensor("op_1713")]; - tensor inputs_97 = mul(x = var_1711_0, y = var_1713)[name = tensor("inputs_97")]; - tensor outputs_aug_29_pad_type_0 = const()[name = tensor("outputs_aug_29_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_29_pad_0 = const()[name = tensor("outputs_aug_29_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_29_groups_0 = const()[name = tensor("outputs_aug_29_groups_0"), val = tensor(256)]; - tensor outputs_aug_29_strides_0 = const()[name = tensor("outputs_aug_29_strides_0"), val = tensor([1])]; - tensor outputs_aug_29_dilations_0 = const()[name = tensor("outputs_aug_29_dilations_0"), val = tensor([1])]; - tensor outputs_aug_29 = conv(dilations = outputs_aug_29_dilations_0, groups = outputs_aug_29_groups_0, pad = outputs_aug_29_pad_0, pad_type = outputs_aug_29_pad_type_0, strides = outputs_aug_29_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_97)[name = tensor("outputs_aug_29")]; - tensor input_225_begin_0 = const()[name = tensor("input_225_begin_0"), val = tensor([0, 0, 0])]; - tensor input_225_end_0 = const()[name = tensor("input_225_end_0"), val = tensor([1, 256, 16])]; - tensor input_225_end_mask_0 = const()[name = tensor("input_225_end_mask_0"), val = tensor([true, true, false])]; - tensor input_225 = slice_by_index(begin = input_225_begin_0, end = input_225_end_0, end_mask = input_225_end_mask_0, x = outputs_aug_29)[name = tensor("input_225")]; - tensor inputs_99 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_225)[name = tensor("inputs_99")]; - tensor input_227 = silu(x = inputs_99)[name = tensor("input_227")]; - tensor input_229_pad_type_0 = const()[name = tensor("input_229_pad_type_0"), val = tensor("valid")]; - tensor input_229_strides_0 = const()[name = tensor("input_229_strides_0"), val = tensor([1])]; - tensor input_229_pad_0 = const()[name = tensor("input_229_pad_0"), val = tensor([0, 0])]; - tensor input_229_dilations_0 = const()[name = tensor("input_229_dilations_0"), val = tensor([1])]; - tensor input_229_groups_0 = const()[name = tensor("input_229_groups_0"), val = tensor(1)]; - tensor input_229 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_229_dilations_0, groups = input_229_groups_0, pad = input_229_pad_0, pad_type = input_229_pad_type_0, strides = input_229_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_227)[name = tensor("input_229")]; - tensor conv_out_29_perm_0 = const()[name = tensor("conv_out_29_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1744_begin_0 = const()[name = tensor("op_1744_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1744_end_0 = const()[name = tensor("op_1744_end_0"), val = tensor([1, 16, 256])]; - tensor var_1744_end_mask_0 = const()[name = tensor("op_1744_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_29 = transpose(perm = conv_out_29_perm_0, x = input_229)[name = tensor("transpose_45")]; - tensor var_1744 = slice_by_index(begin = var_1744_begin_0, end = var_1744_end_0, end_mask = var_1744_end_mask_0, x = conv_out_29)[name = tensor("op_1744")]; - tensor var_1747_interleave_0 = const()[name = tensor("op_1747_interleave_0"), val = tensor(false)]; - tensor var_1747 = concat(axis = var_34, interleave = var_1747_interleave_0, values = (var_1408, var_1492, var_1576, var_1660, var_1744))[name = tensor("op_1747")]; - tensor input_231 = add(x = x_31, y = var_1747)[name = tensor("input_231")]; - tensor input_233_axes_0 = const()[name = tensor("input_233_axes_0"), val = tensor([-1])]; - tensor input_233 = layer_norm(axes = input_233_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_231)[name = tensor("input_233")]; - tensor inputs_101 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_233)[name = tensor("linear_26")]; - tensor input_235 = silu(x = inputs_101)[name = tensor("input_235")]; - tensor input_239 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_235)[name = tensor("linear_27")]; - tensor var_1770 = const()[name = tensor("op_1770"), val = tensor(0x1p-1)]; - tensor var_1771 = mul(x = input_239, y = var_1770)[name = tensor("op_1771")]; - tensor input_241 = add(x = var_1771, y = input_231)[name = tensor("input_241")]; - tensor input_243_axes_0 = const()[name = tensor("input_243_axes_0"), val = tensor([-1])]; - tensor input_243 = layer_norm(axes = input_243_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_36, gamma = encoder_layer_norm_2_weight, x = input_241)[name = tensor("input_243")]; - tensor input_245_axes_0 = const()[name = tensor("input_245_axes_0"), val = tensor([-1])]; - tensor input_245 = layer_norm(axes = input_245_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_243)[name = tensor("input_245")]; - tensor inputs_103 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_245)[name = tensor("linear_28")]; - tensor input_247 = silu(x = inputs_103)[name = tensor("input_247")]; - tensor input_251 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_247)[name = tensor("linear_29")]; - tensor var_1800 = const()[name = tensor("op_1800"), val = tensor(0x1p-1)]; - tensor var_1801 = mul(x = input_251, y = var_1800)[name = tensor("op_1801")]; - tensor input_253 = add(x = var_1801, y = input_243)[name = tensor("input_253")]; - tensor x_43_axes_0 = const()[name = tensor("x_43_axes_0"), val = tensor([-1])]; - tensor x_43 = layer_norm(axes = x_43_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_36, gamma = encoder_ret_lns_3_weight, x = input_253)[name = tensor("x_43")]; + tensor x_expanded_29 = tile(reps = x_expanded_29_reps_0, x = var_1017)[name = tensor("x_expanded_29")]; + tensor var_1031 = sub(x = var_18, y = fill_mask_29)[name = tensor("op_1031")]; + tensor var_1032 = mul(x = window_33, y = var_1031)[name = tensor("op_1032")]; + tensor var_1033 = mul(x = x_expanded_29, y = fill_mask_29)[name = tensor("op_1033")]; + tensor filled_window_29 = add(x = var_1032, y = var_1033)[name = tensor("filled_window_29")]; + tensor var_1035 = greater_equal(x = clip_27, y = var_17)[name = tensor("op_1035")]; + tensor var_1036_dtype_0 = const()[name = tensor("op_1036_dtype_0"), val = tensor("fp32")]; + tensor var_1037 = const()[name = tensor("op_1037"), val = tensor([1, 1, 1])]; + tensor var_1036 = cast(dtype = var_1036_dtype_0, x = var_1035)[name = tensor("cast_162")]; + tensor use_roll_29 = reshape(shape = var_1037, x = var_1036)[name = tensor("use_roll_29")]; + tensor var_1039 = sub(x = var_18, y = use_roll_29)[name = tensor("op_1039")]; + tensor var_1040 = mul(x = filled_window_29, y = var_1039)[name = tensor("op_1040")]; + tensor var_1041 = mul(x = rolled_29, y = use_roll_29)[name = tensor("op_1041")]; + tensor window_35 = add(x = var_1040, y = var_1041)[name = tensor("window_35")]; + tensor var_1043 = const()[name = tensor("op_1043"), val = tensor(0x1p+0)]; + tensor var_1044 = add(x = clip_27, y = var_1043)[name = tensor("op_1044")]; + tensor clip_29 = clip(alpha = var_22, beta = var_17, x = var_1044)[name = tensor("clip_29")]; + tensor input_101_interleave_0 = const()[name = tensor("input_101_interleave_0"), val = tensor(false)]; + tensor input_101 = concat(axis = var_31, interleave = input_101_interleave_0, values = (window_27, window_29, window_31, window_33, window_35))[name = tensor("input_101")]; + tensor x_17_axes_0 = const()[name = tensor("x_17_axes_0"), val = tensor([-1])]; + tensor x_17 = layer_norm(axes = x_17_axes_0, beta = encoder_conv_module_2_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_0_weight, x = input_101)[name = tensor("x_17")]; + tensor input_103_perm_0 = const()[name = tensor("input_103_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_23_pad_type_0 = const()[name = tensor("inputs_23_pad_type_0"), val = tensor("valid")]; + tensor inputs_23_strides_0 = const()[name = tensor("inputs_23_strides_0"), val = tensor([1])]; + tensor inputs_23_pad_0 = const()[name = tensor("inputs_23_pad_0"), val = tensor([0, 0])]; + tensor inputs_23_dilations_0 = const()[name = tensor("inputs_23_dilations_0"), val = tensor([1])]; + tensor inputs_23_groups_0 = const()[name = tensor("inputs_23_groups_0"), val = tensor(1)]; + tensor input_103 = transpose(perm = input_103_perm_0, x = x_17)[name = tensor("transpose_40")]; + tensor inputs_23 = conv(bias = encoder_conv_module_2_sequential_2_conv_bias, dilations = inputs_23_dilations_0, groups = inputs_23_groups_0, pad = inputs_23_pad_0, pad_type = inputs_23_pad_type_0, strides = inputs_23_strides_0, weight = encoder_conv_module_2_sequential_2_conv_weight, x = input_103)[name = tensor("inputs_23")]; + tensor var_1067_split_sizes_0 = const()[name = tensor("op_1067_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1067_axis_0 = const()[name = tensor("op_1067_axis_0"), val = tensor(1)]; + tensor var_1067_0, tensor var_1067_1 = split(axis = var_1067_axis_0, split_sizes = var_1067_split_sizes_0, x = inputs_23)[name = tensor("op_1067")]; + tensor var_1069 = sigmoid(x = var_1067_1)[name = tensor("op_1069")]; + tensor inputs_25 = mul(x = var_1067_0, y = var_1069)[name = tensor("inputs_25")]; + tensor outputs_aug_5_pad_type_0 = const()[name = tensor("outputs_aug_5_pad_type_0"), val = tensor("custom")]; + tensor outputs_aug_5_pad_0 = const()[name = tensor("outputs_aug_5_pad_0"), val = tensor([15, 15])]; + tensor outputs_aug_5_groups_0 = const()[name = tensor("outputs_aug_5_groups_0"), val = tensor(256)]; + tensor outputs_aug_5_strides_0 = const()[name = tensor("outputs_aug_5_strides_0"), val = tensor([1])]; + tensor outputs_aug_5_dilations_0 = const()[name = tensor("outputs_aug_5_dilations_0"), val = tensor([1])]; + tensor outputs_aug_5 = conv(dilations = outputs_aug_5_dilations_0, groups = outputs_aug_5_groups_0, pad = outputs_aug_5_pad_0, pad_type = outputs_aug_5_pad_type_0, strides = outputs_aug_5_strides_0, weight = encoder_conv_module_2_sequential_4_conv_weight, x = inputs_25)[name = tensor("outputs_aug_5")]; + tensor input_105_begin_0 = const()[name = tensor("input_105_begin_0"), val = tensor([0, 0, 0])]; + tensor input_105_end_0 = const()[name = tensor("input_105_end_0"), val = tensor([5, 256, 16])]; + tensor input_105_end_mask_0 = const()[name = tensor("input_105_end_mask_0"), val = tensor([true, true, false])]; + tensor input_105 = slice_by_index(begin = input_105_begin_0, end = input_105_end_0, end_mask = input_105_end_mask_0, x = outputs_aug_5)[name = tensor("input_105")]; + tensor inputs_27 = batch_norm(beta = encoder_conv_module_2_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_2_sequential_5_weight, mean = encoder_conv_module_2_sequential_5_running_mean, variance = encoder_conv_module_2_sequential_5_running_var, x = input_105)[name = tensor("inputs_27")]; + tensor input_107 = silu(x = inputs_27)[name = tensor("input_107")]; + tensor input_109_pad_type_0 = const()[name = tensor("input_109_pad_type_0"), val = tensor("valid")]; + tensor input_109_strides_0 = const()[name = tensor("input_109_strides_0"), val = tensor([1])]; + tensor input_109_pad_0 = const()[name = tensor("input_109_pad_0"), val = tensor([0, 0])]; + tensor input_109_dilations_0 = const()[name = tensor("input_109_dilations_0"), val = tensor([1])]; + tensor input_109_groups_0 = const()[name = tensor("input_109_groups_0"), val = tensor(1)]; + tensor input_109 = conv(bias = encoder_conv_module_2_sequential_7_conv_bias, dilations = input_109_dilations_0, groups = input_109_groups_0, pad = input_109_pad_0, pad_type = input_109_pad_type_0, strides = input_109_strides_0, weight = encoder_conv_module_2_sequential_7_conv_weight, x = input_107)[name = tensor("input_109")]; + tensor conv_out_5_perm_0 = const()[name = tensor("conv_out_5_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1100_begin_0 = const()[name = tensor("op_1100_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1100_end_0 = const()[name = tensor("op_1100_end_0"), val = tensor([5, 16, 256])]; + tensor var_1100_end_mask_0 = const()[name = tensor("op_1100_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_5 = transpose(perm = conv_out_5_perm_0, x = input_109)[name = tensor("transpose_39")]; + tensor var_1100 = slice_by_index(begin = var_1100_begin_0, end = var_1100_end_0, end_mask = var_1100_end_mask_0, x = conv_out_5)[name = tensor("op_1100")]; + tensor var_1102_perm_0 = const()[name = tensor("op_1102_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1102 = transpose(perm = var_1102_perm_0, x = var_1100)[name = tensor("transpose_38")]; + tensor input_111 = add(x = x_15, y = var_1102)[name = tensor("input_111")]; + tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; + tensor input_113 = layer_norm(axes = input_113_axes_0, beta = encoder_ffn2_2_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_2_module_sequential_0_weight, x = input_111)[name = tensor("input_113")]; + tensor inputs_29 = linear(bias = encoder_ffn2_2_module_sequential_1_linear_bias, weight = encoder_ffn2_2_module_sequential_1_linear_weight, x = input_113)[name = tensor("linear_26")]; + tensor input_115 = silu(x = inputs_29)[name = tensor("input_115")]; + tensor input_119 = linear(bias = encoder_ffn2_2_module_sequential_4_linear_bias, weight = encoder_ffn2_2_module_sequential_4_linear_weight, x = input_115)[name = tensor("linear_27")]; + tensor var_1125 = const()[name = tensor("op_1125"), val = tensor(0x1p-1)]; + tensor var_1126 = mul(x = input_119, y = var_1125)[name = tensor("op_1126")]; + tensor input_121 = add(x = var_1126, y = input_111)[name = tensor("input_121")]; + tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; + tensor input_123 = layer_norm(axes = input_123_axes_0, beta = encoder_layer_norm_2_bias, epsilon = var_36, gamma = encoder_layer_norm_2_weight, x = input_121)[name = tensor("input_123")]; + tensor input_125_axes_0 = const()[name = tensor("input_125_axes_0"), val = tensor([-1])]; + tensor input_125 = layer_norm(axes = input_125_axes_0, beta = encoder_ffn1_3_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn1_3_module_sequential_0_weight, x = input_123)[name = tensor("input_125")]; + tensor inputs_31 = linear(bias = encoder_ffn1_3_module_sequential_1_linear_bias, weight = encoder_ffn1_3_module_sequential_1_linear_weight, x = input_125)[name = tensor("linear_28")]; + tensor input_127 = silu(x = inputs_31)[name = tensor("input_127")]; + tensor input_131 = linear(bias = encoder_ffn1_3_module_sequential_4_linear_bias, weight = encoder_ffn1_3_module_sequential_4_linear_weight, x = input_127)[name = tensor("linear_29")]; + tensor var_1155 = const()[name = tensor("op_1155"), val = tensor(0x1p-1)]; + tensor var_1156 = mul(x = input_131, y = var_1155)[name = tensor("op_1156")]; + tensor input_133 = add(x = var_1156, y = input_123)[name = tensor("input_133")]; + tensor x_19_axes_0 = const()[name = tensor("x_19_axes_0"), val = tensor([-1])]; + tensor x_19 = layer_norm(axes = x_19_axes_0, beta = encoder_ret_lns_3_bias, epsilon = var_36, gamma = encoder_ret_lns_3_weight, x = input_133)[name = tensor("x_19")]; tensor prev_kv_7_begin_0 = const()[name = tensor("prev_kv_7_begin_0"), val = tensor([3, 0, 0, 0, 0])]; tensor prev_kv_7_end_0 = const()[name = tensor("prev_kv_7_end_0"), val = tensor([4, 1, 4, 64, 64])]; tensor prev_kv_7_end_mask_0 = const()[name = tensor("prev_kv_7_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -1641,455 +1180,301 @@ program(1.0) tensor prev_scale_7_end_mask_0 = const()[name = tensor("prev_scale_7_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_7_squeeze_mask_0 = const()[name = tensor("prev_scale_7_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_7 = slice_by_index(begin = prev_scale_7_begin_0, end = prev_scale_7_end_0, end_mask = prev_scale_7_end_mask_0, squeeze_mask = prev_scale_7_squeeze_mask_0, x = enc_scale)[name = tensor("prev_scale_7")]; - tensor var_1815 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_43)[name = tensor("linear_30")]; - tensor var_1816 = const()[name = tensor("op_1816"), val = tensor([1, 5, 4, 64])]; - tensor var_1817 = reshape(shape = var_1816, x = var_1815)[name = tensor("op_1817")]; + tensor var_1170 = linear(bias = encoder_q_proj_3_bias, weight = encoder_q_proj_3_weight, x = x_19)[name = tensor("linear_30")]; + tensor var_1171 = const()[name = tensor("op_1171"), val = tensor([1, 5, 4, 64])]; + tensor var_1172 = reshape(shape = var_1171, x = var_1170)[name = tensor("op_1172")]; tensor q_7_perm_0 = const()[name = tensor("q_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1821 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_43)[name = tensor("linear_31")]; - tensor var_1822 = const()[name = tensor("op_1822"), val = tensor(0x1p-3)]; - tensor var_1823 = mul(x = var_1821, y = var_1822)[name = tensor("op_1823")]; - tensor var_1824 = const()[name = tensor("op_1824"), val = tensor([1, 5, 4, 64])]; - tensor var_1825 = reshape(shape = var_1824, x = var_1823)[name = tensor("op_1825")]; + tensor var_1176 = linear(bias = encoder_k_proj_3_bias, weight = encoder_k_proj_3_weight, x = x_19)[name = tensor("linear_31")]; + tensor var_1177 = const()[name = tensor("op_1177"), val = tensor(0x1p-3)]; + tensor var_1178 = mul(x = var_1176, y = var_1177)[name = tensor("op_1178")]; + tensor var_1179 = const()[name = tensor("op_1179"), val = tensor([1, 5, 4, 64])]; + tensor var_1180 = reshape(shape = var_1179, x = var_1178)[name = tensor("op_1180")]; tensor k_7_perm_0 = const()[name = tensor("k_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_1829 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_43)[name = tensor("linear_32")]; - tensor var_1830 = const()[name = tensor("op_1830"), val = tensor([1, 5, 4, 64])]; - tensor var_1831 = reshape(shape = var_1830, x = var_1829)[name = tensor("op_1831")]; + tensor var_1184 = linear(bias = encoder_v_proj_3_bias, weight = encoder_v_proj_3_weight, x = x_19)[name = tensor("linear_32")]; + tensor var_1185 = const()[name = tensor("op_1185"), val = tensor([1, 5, 4, 64])]; + tensor var_1186 = reshape(shape = var_1185, x = var_1184)[name = tensor("op_1186")]; tensor v_7_perm_0 = const()[name = tensor("v_7_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_257 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_43)[name = tensor("linear_33")]; + tensor input_137 = linear(bias = encoder_g_proj_3_bias, weight = encoder_g_proj_3_weight, x = x_19)[name = tensor("linear_33")]; tensor sqrt_s0_7 = sqrt(x = prev_scale_7)[name = tensor("sqrt_s0_7")]; tensor s_t_7 = add(x = prev_scale_7, y = encoder__t_index)[name = tensor("s_t_7")]; tensor sqrt_s_t_7 = sqrt(x = s_t_7)[name = tensor("sqrt_s_t_7")]; tensor qk_7_transpose_x_1 = const()[name = tensor("qk_7_transpose_x_1"), val = tensor(false)]; tensor qk_7_transpose_y_1 = const()[name = tensor("qk_7_transpose_y_1"), val = tensor(true)]; - tensor k_7 = transpose(perm = k_7_perm_0, x = var_1825)[name = tensor("transpose_43")]; - tensor q_7 = transpose(perm = q_7_perm_0, x = var_1817)[name = tensor("transpose_44")]; + tensor k_7 = transpose(perm = k_7_perm_0, x = var_1180)[name = tensor("transpose_36")]; + tensor q_7 = transpose(perm = q_7_perm_0, x = var_1172)[name = tensor("transpose_37")]; tensor qk_7 = matmul(transpose_x = qk_7_transpose_x_1, transpose_y = qk_7_transpose_y_1, x = q_7, y = k_7)[name = tensor("qk_7")]; - tensor var_1841 = const()[name = tensor("op_1841"), val = tensor([5, 1])]; - tensor var_1842 = reshape(shape = var_1841, x = sqrt_s_t_7)[name = tensor("op_1842")]; - tensor M_7 = real_div(x = encoder__causal_mask, y = var_1842)[name = tensor("M_7")]; - tensor var_1844 = mul(x = qk_7, y = M_7)[name = tensor("op_1844")]; + tensor var_1196 = const()[name = tensor("op_1196"), val = tensor([5, 1])]; + tensor var_1197 = reshape(shape = var_1196, x = sqrt_s_t_7)[name = tensor("op_1197")]; + tensor M_7 = real_div(x = encoder__causal_mask, y = var_1197)[name = tensor("M_7")]; + tensor var_1199 = mul(x = qk_7, y = M_7)[name = tensor("op_1199")]; tensor inner_7_transpose_x_0 = const()[name = tensor("inner_7_transpose_x_0"), val = tensor(false)]; tensor inner_7_transpose_y_0 = const()[name = tensor("inner_7_transpose_y_0"), val = tensor(false)]; - tensor v_7 = transpose(perm = v_7_perm_0, x = var_1831)[name = tensor("transpose_42")]; - tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1844, y = v_7)[name = tensor("inner_7")]; - tensor var_1846_transpose_x_0 = const()[name = tensor("op_1846_transpose_x_0"), val = tensor(false)]; - tensor var_1846_transpose_y_0 = const()[name = tensor("op_1846_transpose_y_0"), val = tensor(false)]; - tensor var_1846 = matmul(transpose_x = var_1846_transpose_x_0, transpose_y = var_1846_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1846")]; - tensor var_1847 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1847")]; - tensor var_1848 = const()[name = tensor("op_1848"), val = tensor([1, 1, 5, 1])]; - tensor var_1849 = reshape(shape = var_1848, x = var_1847)[name = tensor("op_1849")]; - tensor cross_7 = mul(x = var_1846, y = var_1849)[name = tensor("cross_7")]; + tensor v_7 = transpose(perm = v_7_perm_0, x = var_1186)[name = tensor("transpose_35")]; + tensor inner_7 = matmul(transpose_x = inner_7_transpose_x_0, transpose_y = inner_7_transpose_y_0, x = var_1199, y = v_7)[name = tensor("inner_7")]; + tensor var_1201_transpose_x_0 = const()[name = tensor("op_1201_transpose_x_0"), val = tensor(false)]; + tensor var_1201_transpose_y_0 = const()[name = tensor("op_1201_transpose_y_0"), val = tensor(false)]; + tensor var_1201 = matmul(transpose_x = var_1201_transpose_x_0, transpose_y = var_1201_transpose_y_0, x = q_7, y = prev_kv_7)[name = tensor("op_1201")]; + tensor var_1202 = real_div(x = sqrt_s0_7, y = sqrt_s_t_7)[name = tensor("op_1202")]; + tensor var_1203 = const()[name = tensor("op_1203"), val = tensor([1, 1, 5, 1])]; + tensor var_1204 = reshape(shape = var_1203, x = var_1202)[name = tensor("op_1204")]; + tensor cross_7 = mul(x = var_1201, y = var_1204)[name = tensor("cross_7")]; tensor out_19 = add(x = inner_7, y = cross_7)[name = tensor("out_19")]; - tensor var_1852 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1852")]; - tensor var_1854_transpose_x_1 = const()[name = tensor("op_1854_transpose_x_1"), val = tensor(true)]; - tensor var_1854_transpose_y_1 = const()[name = tensor("op_1854_transpose_y_1"), val = tensor(false)]; - tensor var_1854 = matmul(transpose_x = var_1854_transpose_x_1, transpose_y = var_1854_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1854")]; - tensor new_kv_unnorm_7 = add(x = var_1852, y = var_1854)[name = tensor("new_kv_unnorm_7")]; - tensor var_1856 = const()[name = tensor("op_1856"), val = tensor(0x1.4p+2)]; - tensor new_scale_7 = add(x = prev_scale_7, y = var_1856)[name = tensor("new_scale_7")]; - tensor var_1858 = sqrt(x = new_scale_7)[name = tensor("op_1858")]; - tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1858)[name = tensor("nkv_1")]; - tensor var_1860_perm_0 = const()[name = tensor("op_1860_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1207 = mul(x = prev_kv_7, y = sqrt_s0_7)[name = tensor("op_1207")]; + tensor var_1209_transpose_x_1 = const()[name = tensor("op_1209_transpose_x_1"), val = tensor(true)]; + tensor var_1209_transpose_y_1 = const()[name = tensor("op_1209_transpose_y_1"), val = tensor(false)]; + tensor var_1209 = matmul(transpose_x = var_1209_transpose_x_1, transpose_y = var_1209_transpose_y_1, x = k_7, y = v_7)[name = tensor("op_1209")]; + tensor new_kv_unnorm_7 = add(x = var_1207, y = var_1209)[name = tensor("new_kv_unnorm_7")]; + tensor var_1211 = const()[name = tensor("op_1211"), val = tensor(0x1.4p+2)]; + tensor new_scale_7 = add(x = prev_scale_7, y = var_1211)[name = tensor("new_scale_7")]; + tensor var_1213 = sqrt(x = new_scale_7)[name = tensor("op_1213")]; + tensor nkv_1 = real_div(x = new_kv_unnorm_7, y = var_1213)[name = tensor("nkv_1")]; + tensor var_1215_perm_0 = const()[name = tensor("op_1215_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_21_axes_0 = const()[name = tensor("out_21_axes_0"), val = tensor([-1])]; - tensor var_1860 = transpose(perm = var_1860_perm_0, x = out_19)[name = tensor("transpose_41")]; - tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_25, x = var_1860)[name = tensor("out_21")]; - tensor var_1864 = const()[name = tensor("op_1864"), val = tensor([1, 5, 256])]; - tensor out_23 = reshape(shape = var_1864, x = out_21)[name = tensor("out_23")]; - tensor var_1866 = silu(x = input_257)[name = tensor("op_1866")]; - tensor input_259 = mul(x = var_1866, y = out_23)[name = tensor("input_259")]; - tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_259)[name = tensor("linear_34")]; - tensor x_45 = add(x = input_253, y = ret_out_7)[name = tensor("x_45")]; - tensor window_begin_0 = const()[name = tensor("window_begin_0"), val = tensor([3, 0, 0, 0])]; - tensor window_end_0 = const()[name = tensor("window_end_0"), val = tensor([4, 1, 16, 256])]; - tensor window_end_mask_0 = const()[name = tensor("window_end_mask_0"), val = tensor([false, true, true, true])]; - tensor window_squeeze_mask_0 = const()[name = tensor("window_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor window = slice_by_index(begin = window_begin_0, end = window_end_0, end_mask = window_end_mask_0, squeeze_mask = window_squeeze_mask_0, x = enc_conv_window)[name = tensor("window")]; + tensor var_1215 = transpose(perm = var_1215_perm_0, x = out_19)[name = tensor("transpose_34")]; + tensor out_21 = layer_norm(axes = out_21_axes_0, epsilon = var_25, x = var_1215)[name = tensor("out_21")]; + tensor var_1219 = const()[name = tensor("op_1219"), val = tensor([1, 5, 256])]; + tensor out_23 = reshape(shape = var_1219, x = out_21)[name = tensor("out_23")]; + tensor var_1221 = silu(x = input_137)[name = tensor("op_1221")]; + tensor input_139 = mul(x = var_1221, y = out_23)[name = tensor("input_139")]; + tensor ret_out_7 = linear(bias = encoder_out_proj_3_bias, weight = encoder_out_proj_3_weight, x = input_139)[name = tensor("linear_34")]; + tensor x_21 = add(x = input_133, y = ret_out_7)[name = tensor("x_21")]; + tensor window_37_begin_0 = const()[name = tensor("window_37_begin_0"), val = tensor([3, 0, 0, 0])]; + tensor window_37_end_0 = const()[name = tensor("window_37_end_0"), val = tensor([4, 1, 16, 256])]; + tensor window_37_end_mask_0 = const()[name = tensor("window_37_end_mask_0"), val = tensor([false, true, true, true])]; + tensor window_37_squeeze_mask_0 = const()[name = tensor("window_37_squeeze_mask_0"), val = tensor([true, false, false, false])]; + tensor window_37 = slice_by_index(begin = window_37_begin_0, end = window_37_end_0, end_mask = window_37_end_mask_0, squeeze_mask = window_37_squeeze_mask_0, x = enc_conv_window)[name = tensor("window_37")]; tensor filled_31_begin_0 = const()[name = tensor("filled_31_begin_0"), val = tensor([3, 0])]; tensor filled_31_end_0 = const()[name = tensor("filled_31_end_0"), val = tensor([4, 1])]; tensor filled_31_end_mask_0 = const()[name = tensor("filled_31_end_mask_0"), val = tensor([false, true])]; tensor filled_31_squeeze_mask_0 = const()[name = tensor("filled_31_squeeze_mask_0"), val = tensor([true, false])]; tensor filled_31 = slice_by_index(begin = filled_31_begin_0, end = filled_31_end_0, end_mask = filled_31_end_mask_0, squeeze_mask = filled_31_squeeze_mask_0, x = enc_conv_filled)[name = tensor("filled_31")]; - tensor var_1875_begin_0 = const()[name = tensor("op_1875_begin_0"), val = tensor([0, 0, 0])]; - tensor var_1875_end_0 = const()[name = tensor("op_1875_end_0"), val = tensor([1, 1, 256])]; - tensor var_1875_end_mask_0 = const()[name = tensor("op_1875_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1875 = slice_by_index(begin = var_1875_begin_0, end = var_1875_end_0, end_mask = var_1875_end_mask_0, x = x_45)[name = tensor("op_1875")]; - tensor var_1878_begin_0 = const()[name = tensor("op_1878_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1878_end_0 = const()[name = tensor("op_1878_end_0"), val = tensor([1, 16, 256])]; - tensor var_1878_end_mask_0 = const()[name = tensor("op_1878_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1878 = slice_by_index(begin = var_1878_begin_0, end = var_1878_end_0, end_mask = var_1878_end_mask_0, x = window)[name = tensor("op_1878")]; + tensor var_1230_begin_0 = const()[name = tensor("op_1230_begin_0"), val = tensor([0, 0, 0])]; + tensor var_1230_end_0 = const()[name = tensor("op_1230_end_0"), val = tensor([1, 1, 256])]; + tensor var_1230_end_mask_0 = const()[name = tensor("op_1230_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1230 = slice_by_index(begin = var_1230_begin_0, end = var_1230_end_0, end_mask = var_1230_end_mask_0, x = x_21)[name = tensor("op_1230")]; + tensor var_1233_begin_0 = const()[name = tensor("op_1233_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1233_end_0 = const()[name = tensor("op_1233_end_0"), val = tensor([1, 16, 256])]; + tensor var_1233_end_mask_0 = const()[name = tensor("op_1233_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1233 = slice_by_index(begin = var_1233_begin_0, end = var_1233_end_0, end_mask = var_1233_end_mask_0, x = window_37)[name = tensor("op_1233")]; tensor rolled_31_interleave_0 = const()[name = tensor("rolled_31_interleave_0"), val = tensor(false)]; - tensor rolled_31 = concat(axis = var_34, interleave = rolled_31_interleave_0, values = (var_1878, var_1875))[name = tensor("rolled_31")]; + tensor rolled_31 = concat(axis = var_34, interleave = rolled_31_interleave_0, values = (var_1233, var_1230))[name = tensor("rolled_31")]; tensor clip_30 = clip(alpha = var_22, beta = var_21, x = filled_31)[name = tensor("clip_30")]; - tensor var_1883 = equal(x = encoder__conv_positions, y = clip_30)[name = tensor("op_1883")]; - tensor var_1884_dtype_0 = const()[name = tensor("op_1884_dtype_0"), val = tensor("fp32")]; - tensor var_1885 = const()[name = tensor("op_1885"), val = tensor([1, 16, 1])]; - tensor var_1884 = cast(dtype = var_1884_dtype_0, x = var_1883)[name = tensor("cast_161")]; - tensor fill_mask_31 = reshape(shape = var_1885, x = var_1884)[name = tensor("fill_mask_31")]; + tensor var_1238 = equal(x = encoder__conv_positions, y = clip_30)[name = tensor("op_1238")]; + tensor var_1239_dtype_0 = const()[name = tensor("op_1239_dtype_0"), val = tensor("fp32")]; + tensor var_1240 = const()[name = tensor("op_1240"), val = tensor([1, 16, 1])]; + tensor var_1239 = cast(dtype = var_1239_dtype_0, x = var_1238)[name = tensor("cast_161")]; + tensor fill_mask_31 = reshape(shape = var_1240, x = var_1239)[name = tensor("fill_mask_31")]; tensor x_expanded_31_reps_0 = const()[name = tensor("x_expanded_31_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_31 = tile(reps = x_expanded_31_reps_0, x = var_1875)[name = tensor("x_expanded_31")]; - tensor var_1889 = sub(x = var_18, y = fill_mask_31)[name = tensor("op_1889")]; - tensor var_1890 = mul(x = window, y = var_1889)[name = tensor("op_1890")]; - tensor var_1891 = mul(x = x_expanded_31, y = fill_mask_31)[name = tensor("op_1891")]; - tensor filled_window_31 = add(x = var_1890, y = var_1891)[name = tensor("filled_window_31")]; - tensor var_1893 = greater_equal(x = filled_31, y = var_17)[name = tensor("op_1893")]; - tensor var_1894_dtype_0 = const()[name = tensor("op_1894_dtype_0"), val = tensor("fp32")]; - tensor var_1895 = const()[name = tensor("op_1895"), val = tensor([1, 1, 1])]; - tensor var_1894 = cast(dtype = var_1894_dtype_0, x = var_1893)[name = tensor("cast_160")]; - tensor use_roll_31 = reshape(shape = var_1895, x = var_1894)[name = tensor("use_roll_31")]; - tensor var_1897 = sub(x = var_18, y = use_roll_31)[name = tensor("op_1897")]; - tensor var_1898 = mul(x = filled_window_31, y = var_1897)[name = tensor("op_1898")]; - tensor var_1899 = mul(x = rolled_31, y = use_roll_31)[name = tensor("op_1899")]; - tensor input_261 = add(x = var_1898, y = var_1899)[name = tensor("input_261")]; - tensor var_1901 = const()[name = tensor("op_1901"), val = tensor(0x1p+0)]; - tensor var_1902 = add(x = filled_31, y = var_1901)[name = tensor("op_1902")]; - tensor clip_31 = clip(alpha = var_22, beta = var_17, x = var_1902)[name = tensor("clip_31")]; - tensor x_47_axes_0 = const()[name = tensor("x_47_axes_0"), val = tensor([-1])]; - tensor x_47 = layer_norm(axes = x_47_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_261)[name = tensor("x_47")]; - tensor input_263_perm_0 = const()[name = tensor("input_263_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_105_pad_type_0 = const()[name = tensor("inputs_105_pad_type_0"), val = tensor("valid")]; - tensor inputs_105_strides_0 = const()[name = tensor("inputs_105_strides_0"), val = tensor([1])]; - tensor inputs_105_pad_0 = const()[name = tensor("inputs_105_pad_0"), val = tensor([0, 0])]; - tensor inputs_105_dilations_0 = const()[name = tensor("inputs_105_dilations_0"), val = tensor([1])]; - tensor inputs_105_groups_0 = const()[name = tensor("inputs_105_groups_0"), val = tensor(1)]; - tensor input_263 = transpose(perm = input_263_perm_0, x = x_47)[name = tensor("transpose_40")]; - tensor inputs_105 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_105_dilations_0, groups = inputs_105_groups_0, pad = inputs_105_pad_0, pad_type = inputs_105_pad_type_0, strides = inputs_105_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_263)[name = tensor("inputs_105")]; - tensor var_1923_split_sizes_0 = const()[name = tensor("op_1923_split_sizes_0"), val = tensor([256, 256])]; - tensor var_1923_axis_0 = const()[name = tensor("op_1923_axis_0"), val = tensor(1)]; - tensor var_1923_0, tensor var_1923_1 = split(axis = var_1923_axis_0, split_sizes = var_1923_split_sizes_0, x = inputs_105)[name = tensor("op_1923")]; - tensor var_1925 = sigmoid(x = var_1923_1)[name = tensor("op_1925")]; - tensor inputs_107 = mul(x = var_1923_0, y = var_1925)[name = tensor("inputs_107")]; - tensor outputs_aug_31_pad_type_0 = const()[name = tensor("outputs_aug_31_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_31_pad_0 = const()[name = tensor("outputs_aug_31_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_31_groups_0 = const()[name = tensor("outputs_aug_31_groups_0"), val = tensor(256)]; - tensor outputs_aug_31_strides_0 = const()[name = tensor("outputs_aug_31_strides_0"), val = tensor([1])]; - tensor outputs_aug_31_dilations_0 = const()[name = tensor("outputs_aug_31_dilations_0"), val = tensor([1])]; - tensor outputs_aug_31 = conv(dilations = outputs_aug_31_dilations_0, groups = outputs_aug_31_groups_0, pad = outputs_aug_31_pad_0, pad_type = outputs_aug_31_pad_type_0, strides = outputs_aug_31_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_107)[name = tensor("outputs_aug_31")]; - tensor input_265_begin_0 = const()[name = tensor("input_265_begin_0"), val = tensor([0, 0, 0])]; - tensor input_265_end_0 = const()[name = tensor("input_265_end_0"), val = tensor([1, 256, 16])]; - tensor input_265_end_mask_0 = const()[name = tensor("input_265_end_mask_0"), val = tensor([true, true, false])]; - tensor input_265 = slice_by_index(begin = input_265_begin_0, end = input_265_end_0, end_mask = input_265_end_mask_0, x = outputs_aug_31)[name = tensor("input_265")]; - tensor inputs_109 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_265)[name = tensor("inputs_109")]; - tensor input_267 = silu(x = inputs_109)[name = tensor("input_267")]; - tensor input_269_pad_type_0 = const()[name = tensor("input_269_pad_type_0"), val = tensor("valid")]; - tensor input_269_strides_0 = const()[name = tensor("input_269_strides_0"), val = tensor([1])]; - tensor input_269_pad_0 = const()[name = tensor("input_269_pad_0"), val = tensor([0, 0])]; - tensor input_269_dilations_0 = const()[name = tensor("input_269_dilations_0"), val = tensor([1])]; - tensor input_269_groups_0 = const()[name = tensor("input_269_groups_0"), val = tensor(1)]; - tensor input_269 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_269_dilations_0, groups = input_269_groups_0, pad = input_269_pad_0, pad_type = input_269_pad_type_0, strides = input_269_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_267)[name = tensor("input_269")]; - tensor conv_out_31_perm_0 = const()[name = tensor("conv_out_31_perm_0"), val = tensor([0, 2, 1])]; - tensor var_1956_begin_0 = const()[name = tensor("op_1956_begin_0"), val = tensor([0, -1, 0])]; - tensor var_1956_end_0 = const()[name = tensor("op_1956_end_0"), val = tensor([1, 16, 256])]; - tensor var_1956_end_mask_0 = const()[name = tensor("op_1956_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_31 = transpose(perm = conv_out_31_perm_0, x = input_269)[name = tensor("transpose_39")]; - tensor var_1956 = slice_by_index(begin = var_1956_begin_0, end = var_1956_end_0, end_mask = var_1956_end_mask_0, x = conv_out_31)[name = tensor("op_1956")]; - tensor var_1959_begin_0 = const()[name = tensor("op_1959_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1959_end_0 = const()[name = tensor("op_1959_end_0"), val = tensor([1, 2, 256])]; - tensor var_1959_end_mask_0 = const()[name = tensor("op_1959_end_mask_0"), val = tensor([true, false, true])]; - tensor var_1959 = slice_by_index(begin = var_1959_begin_0, end = var_1959_end_0, end_mask = var_1959_end_mask_0, x = x_45)[name = tensor("op_1959")]; - tensor var_1962_begin_0 = const()[name = tensor("op_1962_begin_0"), val = tensor([0, 1, 0])]; - tensor var_1962_end_0 = const()[name = tensor("op_1962_end_0"), val = tensor([1, 16, 256])]; - tensor var_1962_end_mask_0 = const()[name = tensor("op_1962_end_mask_0"), val = tensor([true, true, true])]; - tensor var_1962 = slice_by_index(begin = var_1962_begin_0, end = var_1962_end_0, end_mask = var_1962_end_mask_0, x = input_261)[name = tensor("op_1962")]; + tensor x_expanded_31 = tile(reps = x_expanded_31_reps_0, x = var_1230)[name = tensor("x_expanded_31")]; + tensor var_1244 = sub(x = var_18, y = fill_mask_31)[name = tensor("op_1244")]; + tensor var_1245 = mul(x = window_37, y = var_1244)[name = tensor("op_1245")]; + tensor var_1246 = mul(x = x_expanded_31, y = fill_mask_31)[name = tensor("op_1246")]; + tensor filled_window_31 = add(x = var_1245, y = var_1246)[name = tensor("filled_window_31")]; + tensor var_1248 = greater_equal(x = filled_31, y = var_17)[name = tensor("op_1248")]; + tensor var_1249_dtype_0 = const()[name = tensor("op_1249_dtype_0"), val = tensor("fp32")]; + tensor var_1250 = const()[name = tensor("op_1250"), val = tensor([1, 1, 1])]; + tensor var_1249 = cast(dtype = var_1249_dtype_0, x = var_1248)[name = tensor("cast_160")]; + tensor use_roll_31 = reshape(shape = var_1250, x = var_1249)[name = tensor("use_roll_31")]; + tensor var_1252 = sub(x = var_18, y = use_roll_31)[name = tensor("op_1252")]; + tensor var_1253 = mul(x = filled_window_31, y = var_1252)[name = tensor("op_1253")]; + tensor var_1254 = mul(x = rolled_31, y = use_roll_31)[name = tensor("op_1254")]; + tensor window_39 = add(x = var_1253, y = var_1254)[name = tensor("window_39")]; + tensor var_1256 = const()[name = tensor("op_1256"), val = tensor(0x1p+0)]; + tensor var_1257 = add(x = filled_31, y = var_1256)[name = tensor("op_1257")]; + tensor clip_31 = clip(alpha = var_22, beta = var_17, x = var_1257)[name = tensor("clip_31")]; + tensor var_1260_begin_0 = const()[name = tensor("op_1260_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1260_end_0 = const()[name = tensor("op_1260_end_0"), val = tensor([1, 2, 256])]; + tensor var_1260_end_mask_0 = const()[name = tensor("op_1260_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1260 = slice_by_index(begin = var_1260_begin_0, end = var_1260_end_0, end_mask = var_1260_end_mask_0, x = x_21)[name = tensor("op_1260")]; + tensor var_1263_begin_0 = const()[name = tensor("op_1263_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1263_end_0 = const()[name = tensor("op_1263_end_0"), val = tensor([1, 16, 256])]; + tensor var_1263_end_mask_0 = const()[name = tensor("op_1263_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1263 = slice_by_index(begin = var_1263_begin_0, end = var_1263_end_0, end_mask = var_1263_end_mask_0, x = window_39)[name = tensor("op_1263")]; tensor rolled_33_interleave_0 = const()[name = tensor("rolled_33_interleave_0"), val = tensor(false)]; - tensor rolled_33 = concat(axis = var_34, interleave = rolled_33_interleave_0, values = (var_1962, var_1959))[name = tensor("rolled_33")]; + tensor rolled_33 = concat(axis = var_34, interleave = rolled_33_interleave_0, values = (var_1263, var_1260))[name = tensor("rolled_33")]; tensor clip_32 = clip(alpha = var_22, beta = var_21, x = clip_31)[name = tensor("clip_32")]; - tensor var_1967 = equal(x = encoder__conv_positions, y = clip_32)[name = tensor("op_1967")]; - tensor var_1968_dtype_0 = const()[name = tensor("op_1968_dtype_0"), val = tensor("fp32")]; - tensor var_1969 = const()[name = tensor("op_1969"), val = tensor([1, 16, 1])]; - tensor var_1968 = cast(dtype = var_1968_dtype_0, x = var_1967)[name = tensor("cast_159")]; - tensor fill_mask_33 = reshape(shape = var_1969, x = var_1968)[name = tensor("fill_mask_33")]; + tensor var_1268 = equal(x = encoder__conv_positions, y = clip_32)[name = tensor("op_1268")]; + tensor var_1269_dtype_0 = const()[name = tensor("op_1269_dtype_0"), val = tensor("fp32")]; + tensor var_1270 = const()[name = tensor("op_1270"), val = tensor([1, 16, 1])]; + tensor var_1269 = cast(dtype = var_1269_dtype_0, x = var_1268)[name = tensor("cast_159")]; + tensor fill_mask_33 = reshape(shape = var_1270, x = var_1269)[name = tensor("fill_mask_33")]; tensor x_expanded_33_reps_0 = const()[name = tensor("x_expanded_33_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_33 = tile(reps = x_expanded_33_reps_0, x = var_1959)[name = tensor("x_expanded_33")]; - tensor var_1973 = sub(x = var_18, y = fill_mask_33)[name = tensor("op_1973")]; - tensor var_1974 = mul(x = input_261, y = var_1973)[name = tensor("op_1974")]; - tensor var_1975 = mul(x = x_expanded_33, y = fill_mask_33)[name = tensor("op_1975")]; - tensor filled_window_33 = add(x = var_1974, y = var_1975)[name = tensor("filled_window_33")]; - tensor var_1977 = greater_equal(x = clip_31, y = var_17)[name = tensor("op_1977")]; - tensor var_1978_dtype_0 = const()[name = tensor("op_1978_dtype_0"), val = tensor("fp32")]; - tensor var_1979 = const()[name = tensor("op_1979"), val = tensor([1, 1, 1])]; - tensor var_1978 = cast(dtype = var_1978_dtype_0, x = var_1977)[name = tensor("cast_158")]; - tensor use_roll_33 = reshape(shape = var_1979, x = var_1978)[name = tensor("use_roll_33")]; - tensor var_1981 = sub(x = var_18, y = use_roll_33)[name = tensor("op_1981")]; - tensor var_1982 = mul(x = filled_window_33, y = var_1981)[name = tensor("op_1982")]; - tensor var_1983 = mul(x = rolled_33, y = use_roll_33)[name = tensor("op_1983")]; - tensor input_271 = add(x = var_1982, y = var_1983)[name = tensor("input_271")]; - tensor var_1985 = const()[name = tensor("op_1985"), val = tensor(0x1p+0)]; - tensor var_1986 = add(x = clip_31, y = var_1985)[name = tensor("op_1986")]; - tensor clip_33 = clip(alpha = var_22, beta = var_17, x = var_1986)[name = tensor("clip_33")]; - tensor x_49_axes_0 = const()[name = tensor("x_49_axes_0"), val = tensor([-1])]; - tensor x_49 = layer_norm(axes = x_49_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_271)[name = tensor("x_49")]; - tensor input_273_perm_0 = const()[name = tensor("input_273_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_111_pad_type_0 = const()[name = tensor("inputs_111_pad_type_0"), val = tensor("valid")]; - tensor inputs_111_strides_0 = const()[name = tensor("inputs_111_strides_0"), val = tensor([1])]; - tensor inputs_111_pad_0 = const()[name = tensor("inputs_111_pad_0"), val = tensor([0, 0])]; - tensor inputs_111_dilations_0 = const()[name = tensor("inputs_111_dilations_0"), val = tensor([1])]; - tensor inputs_111_groups_0 = const()[name = tensor("inputs_111_groups_0"), val = tensor(1)]; - tensor input_273 = transpose(perm = input_273_perm_0, x = x_49)[name = tensor("transpose_38")]; - tensor inputs_111 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_111_dilations_0, groups = inputs_111_groups_0, pad = inputs_111_pad_0, pad_type = inputs_111_pad_type_0, strides = inputs_111_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_273)[name = tensor("inputs_111")]; - tensor var_2007_split_sizes_0 = const()[name = tensor("op_2007_split_sizes_0"), val = tensor([256, 256])]; - tensor var_2007_axis_0 = const()[name = tensor("op_2007_axis_0"), val = tensor(1)]; - tensor var_2007_0, tensor var_2007_1 = split(axis = var_2007_axis_0, split_sizes = var_2007_split_sizes_0, x = inputs_111)[name = tensor("op_2007")]; - tensor var_2009 = sigmoid(x = var_2007_1)[name = tensor("op_2009")]; - tensor inputs_113 = mul(x = var_2007_0, y = var_2009)[name = tensor("inputs_113")]; - tensor outputs_aug_33_pad_type_0 = const()[name = tensor("outputs_aug_33_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_33_pad_0 = const()[name = tensor("outputs_aug_33_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_33_groups_0 = const()[name = tensor("outputs_aug_33_groups_0"), val = tensor(256)]; - tensor outputs_aug_33_strides_0 = const()[name = tensor("outputs_aug_33_strides_0"), val = tensor([1])]; - tensor outputs_aug_33_dilations_0 = const()[name = tensor("outputs_aug_33_dilations_0"), val = tensor([1])]; - tensor outputs_aug_33 = conv(dilations = outputs_aug_33_dilations_0, groups = outputs_aug_33_groups_0, pad = outputs_aug_33_pad_0, pad_type = outputs_aug_33_pad_type_0, strides = outputs_aug_33_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_113)[name = tensor("outputs_aug_33")]; - tensor input_275_begin_0 = const()[name = tensor("input_275_begin_0"), val = tensor([0, 0, 0])]; - tensor input_275_end_0 = const()[name = tensor("input_275_end_0"), val = tensor([1, 256, 16])]; - tensor input_275_end_mask_0 = const()[name = tensor("input_275_end_mask_0"), val = tensor([true, true, false])]; - tensor input_275 = slice_by_index(begin = input_275_begin_0, end = input_275_end_0, end_mask = input_275_end_mask_0, x = outputs_aug_33)[name = tensor("input_275")]; - tensor inputs_115 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_275)[name = tensor("inputs_115")]; - tensor input_277 = silu(x = inputs_115)[name = tensor("input_277")]; - tensor input_279_pad_type_0 = const()[name = tensor("input_279_pad_type_0"), val = tensor("valid")]; - tensor input_279_strides_0 = const()[name = tensor("input_279_strides_0"), val = tensor([1])]; - tensor input_279_pad_0 = const()[name = tensor("input_279_pad_0"), val = tensor([0, 0])]; - tensor input_279_dilations_0 = const()[name = tensor("input_279_dilations_0"), val = tensor([1])]; - tensor input_279_groups_0 = const()[name = tensor("input_279_groups_0"), val = tensor(1)]; - tensor input_279 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_279_dilations_0, groups = input_279_groups_0, pad = input_279_pad_0, pad_type = input_279_pad_type_0, strides = input_279_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_277)[name = tensor("input_279")]; - tensor conv_out_33_perm_0 = const()[name = tensor("conv_out_33_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2040_begin_0 = const()[name = tensor("op_2040_begin_0"), val = tensor([0, -1, 0])]; - tensor var_2040_end_0 = const()[name = tensor("op_2040_end_0"), val = tensor([1, 16, 256])]; - tensor var_2040_end_mask_0 = const()[name = tensor("op_2040_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_33 = transpose(perm = conv_out_33_perm_0, x = input_279)[name = tensor("transpose_37")]; - tensor var_2040 = slice_by_index(begin = var_2040_begin_0, end = var_2040_end_0, end_mask = var_2040_end_mask_0, x = conv_out_33)[name = tensor("op_2040")]; - tensor var_2043_begin_0 = const()[name = tensor("op_2043_begin_0"), val = tensor([0, 2, 0])]; - tensor var_2043_end_0 = const()[name = tensor("op_2043_end_0"), val = tensor([1, 3, 256])]; - tensor var_2043_end_mask_0 = const()[name = tensor("op_2043_end_mask_0"), val = tensor([true, false, true])]; - tensor var_2043 = slice_by_index(begin = var_2043_begin_0, end = var_2043_end_0, end_mask = var_2043_end_mask_0, x = x_45)[name = tensor("op_2043")]; - tensor var_2046_begin_0 = const()[name = tensor("op_2046_begin_0"), val = tensor([0, 1, 0])]; - tensor var_2046_end_0 = const()[name = tensor("op_2046_end_0"), val = tensor([1, 16, 256])]; - tensor var_2046_end_mask_0 = const()[name = tensor("op_2046_end_mask_0"), val = tensor([true, true, true])]; - tensor var_2046 = slice_by_index(begin = var_2046_begin_0, end = var_2046_end_0, end_mask = var_2046_end_mask_0, x = input_271)[name = tensor("op_2046")]; + tensor x_expanded_33 = tile(reps = x_expanded_33_reps_0, x = var_1260)[name = tensor("x_expanded_33")]; + tensor var_1274 = sub(x = var_18, y = fill_mask_33)[name = tensor("op_1274")]; + tensor var_1275 = mul(x = window_39, y = var_1274)[name = tensor("op_1275")]; + tensor var_1276 = mul(x = x_expanded_33, y = fill_mask_33)[name = tensor("op_1276")]; + tensor filled_window_33 = add(x = var_1275, y = var_1276)[name = tensor("filled_window_33")]; + tensor var_1278 = greater_equal(x = clip_31, y = var_17)[name = tensor("op_1278")]; + tensor var_1279_dtype_0 = const()[name = tensor("op_1279_dtype_0"), val = tensor("fp32")]; + tensor var_1280 = const()[name = tensor("op_1280"), val = tensor([1, 1, 1])]; + tensor var_1279 = cast(dtype = var_1279_dtype_0, x = var_1278)[name = tensor("cast_158")]; + tensor use_roll_33 = reshape(shape = var_1280, x = var_1279)[name = tensor("use_roll_33")]; + tensor var_1282 = sub(x = var_18, y = use_roll_33)[name = tensor("op_1282")]; + tensor var_1283 = mul(x = filled_window_33, y = var_1282)[name = tensor("op_1283")]; + tensor var_1284 = mul(x = rolled_33, y = use_roll_33)[name = tensor("op_1284")]; + tensor window_41 = add(x = var_1283, y = var_1284)[name = tensor("window_41")]; + tensor var_1286 = const()[name = tensor("op_1286"), val = tensor(0x1p+0)]; + tensor var_1287 = add(x = clip_31, y = var_1286)[name = tensor("op_1287")]; + tensor clip_33 = clip(alpha = var_22, beta = var_17, x = var_1287)[name = tensor("clip_33")]; + tensor var_1290_begin_0 = const()[name = tensor("op_1290_begin_0"), val = tensor([0, 2, 0])]; + tensor var_1290_end_0 = const()[name = tensor("op_1290_end_0"), val = tensor([1, 3, 256])]; + tensor var_1290_end_mask_0 = const()[name = tensor("op_1290_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1290 = slice_by_index(begin = var_1290_begin_0, end = var_1290_end_0, end_mask = var_1290_end_mask_0, x = x_21)[name = tensor("op_1290")]; + tensor var_1293_begin_0 = const()[name = tensor("op_1293_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1293_end_0 = const()[name = tensor("op_1293_end_0"), val = tensor([1, 16, 256])]; + tensor var_1293_end_mask_0 = const()[name = tensor("op_1293_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1293 = slice_by_index(begin = var_1293_begin_0, end = var_1293_end_0, end_mask = var_1293_end_mask_0, x = window_41)[name = tensor("op_1293")]; tensor rolled_35_interleave_0 = const()[name = tensor("rolled_35_interleave_0"), val = tensor(false)]; - tensor rolled_35 = concat(axis = var_34, interleave = rolled_35_interleave_0, values = (var_2046, var_2043))[name = tensor("rolled_35")]; + tensor rolled_35 = concat(axis = var_34, interleave = rolled_35_interleave_0, values = (var_1293, var_1290))[name = tensor("rolled_35")]; tensor clip_34 = clip(alpha = var_22, beta = var_21, x = clip_33)[name = tensor("clip_34")]; - tensor var_2051 = equal(x = encoder__conv_positions, y = clip_34)[name = tensor("op_2051")]; - tensor var_2052_dtype_0 = const()[name = tensor("op_2052_dtype_0"), val = tensor("fp32")]; - tensor var_2053 = const()[name = tensor("op_2053"), val = tensor([1, 16, 1])]; - tensor var_2052 = cast(dtype = var_2052_dtype_0, x = var_2051)[name = tensor("cast_157")]; - tensor fill_mask_35 = reshape(shape = var_2053, x = var_2052)[name = tensor("fill_mask_35")]; + tensor var_1298 = equal(x = encoder__conv_positions, y = clip_34)[name = tensor("op_1298")]; + tensor var_1299_dtype_0 = const()[name = tensor("op_1299_dtype_0"), val = tensor("fp32")]; + tensor var_1300 = const()[name = tensor("op_1300"), val = tensor([1, 16, 1])]; + tensor var_1299 = cast(dtype = var_1299_dtype_0, x = var_1298)[name = tensor("cast_157")]; + tensor fill_mask_35 = reshape(shape = var_1300, x = var_1299)[name = tensor("fill_mask_35")]; tensor x_expanded_35_reps_0 = const()[name = tensor("x_expanded_35_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_35 = tile(reps = x_expanded_35_reps_0, x = var_2043)[name = tensor("x_expanded_35")]; - tensor var_2057 = sub(x = var_18, y = fill_mask_35)[name = tensor("op_2057")]; - tensor var_2058 = mul(x = input_271, y = var_2057)[name = tensor("op_2058")]; - tensor var_2059 = mul(x = x_expanded_35, y = fill_mask_35)[name = tensor("op_2059")]; - tensor filled_window_35 = add(x = var_2058, y = var_2059)[name = tensor("filled_window_35")]; - tensor var_2061 = greater_equal(x = clip_33, y = var_17)[name = tensor("op_2061")]; - tensor var_2062_dtype_0 = const()[name = tensor("op_2062_dtype_0"), val = tensor("fp32")]; - tensor var_2063 = const()[name = tensor("op_2063"), val = tensor([1, 1, 1])]; - tensor var_2062 = cast(dtype = var_2062_dtype_0, x = var_2061)[name = tensor("cast_156")]; - tensor use_roll_35 = reshape(shape = var_2063, x = var_2062)[name = tensor("use_roll_35")]; - tensor var_2065 = sub(x = var_18, y = use_roll_35)[name = tensor("op_2065")]; - tensor var_2066 = mul(x = filled_window_35, y = var_2065)[name = tensor("op_2066")]; - tensor var_2067 = mul(x = rolled_35, y = use_roll_35)[name = tensor("op_2067")]; - tensor input_281 = add(x = var_2066, y = var_2067)[name = tensor("input_281")]; - tensor var_2069 = const()[name = tensor("op_2069"), val = tensor(0x1p+0)]; - tensor var_2070 = add(x = clip_33, y = var_2069)[name = tensor("op_2070")]; - tensor clip_35 = clip(alpha = var_22, beta = var_17, x = var_2070)[name = tensor("clip_35")]; - tensor x_51_axes_0 = const()[name = tensor("x_51_axes_0"), val = tensor([-1])]; - tensor x_51 = layer_norm(axes = x_51_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_281)[name = tensor("x_51")]; - tensor input_283_perm_0 = const()[name = tensor("input_283_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_117_pad_type_0 = const()[name = tensor("inputs_117_pad_type_0"), val = tensor("valid")]; - tensor inputs_117_strides_0 = const()[name = tensor("inputs_117_strides_0"), val = tensor([1])]; - tensor inputs_117_pad_0 = const()[name = tensor("inputs_117_pad_0"), val = tensor([0, 0])]; - tensor inputs_117_dilations_0 = const()[name = tensor("inputs_117_dilations_0"), val = tensor([1])]; - tensor inputs_117_groups_0 = const()[name = tensor("inputs_117_groups_0"), val = tensor(1)]; - tensor input_283 = transpose(perm = input_283_perm_0, x = x_51)[name = tensor("transpose_36")]; - tensor inputs_117 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_117_dilations_0, groups = inputs_117_groups_0, pad = inputs_117_pad_0, pad_type = inputs_117_pad_type_0, strides = inputs_117_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_283)[name = tensor("inputs_117")]; - tensor var_2091_split_sizes_0 = const()[name = tensor("op_2091_split_sizes_0"), val = tensor([256, 256])]; - tensor var_2091_axis_0 = const()[name = tensor("op_2091_axis_0"), val = tensor(1)]; - tensor var_2091_0, tensor var_2091_1 = split(axis = var_2091_axis_0, split_sizes = var_2091_split_sizes_0, x = inputs_117)[name = tensor("op_2091")]; - tensor var_2093 = sigmoid(x = var_2091_1)[name = tensor("op_2093")]; - tensor inputs_119 = mul(x = var_2091_0, y = var_2093)[name = tensor("inputs_119")]; - tensor outputs_aug_35_pad_type_0 = const()[name = tensor("outputs_aug_35_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_35_pad_0 = const()[name = tensor("outputs_aug_35_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_35_groups_0 = const()[name = tensor("outputs_aug_35_groups_0"), val = tensor(256)]; - tensor outputs_aug_35_strides_0 = const()[name = tensor("outputs_aug_35_strides_0"), val = tensor([1])]; - tensor outputs_aug_35_dilations_0 = const()[name = tensor("outputs_aug_35_dilations_0"), val = tensor([1])]; - tensor outputs_aug_35 = conv(dilations = outputs_aug_35_dilations_0, groups = outputs_aug_35_groups_0, pad = outputs_aug_35_pad_0, pad_type = outputs_aug_35_pad_type_0, strides = outputs_aug_35_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_119)[name = tensor("outputs_aug_35")]; - tensor input_285_begin_0 = const()[name = tensor("input_285_begin_0"), val = tensor([0, 0, 0])]; - tensor input_285_end_0 = const()[name = tensor("input_285_end_0"), val = tensor([1, 256, 16])]; - tensor input_285_end_mask_0 = const()[name = tensor("input_285_end_mask_0"), val = tensor([true, true, false])]; - tensor input_285 = slice_by_index(begin = input_285_begin_0, end = input_285_end_0, end_mask = input_285_end_mask_0, x = outputs_aug_35)[name = tensor("input_285")]; - tensor inputs_121 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_285)[name = tensor("inputs_121")]; - tensor input_287 = silu(x = inputs_121)[name = tensor("input_287")]; - tensor input_289_pad_type_0 = const()[name = tensor("input_289_pad_type_0"), val = tensor("valid")]; - tensor input_289_strides_0 = const()[name = tensor("input_289_strides_0"), val = tensor([1])]; - tensor input_289_pad_0 = const()[name = tensor("input_289_pad_0"), val = tensor([0, 0])]; - tensor input_289_dilations_0 = const()[name = tensor("input_289_dilations_0"), val = tensor([1])]; - tensor input_289_groups_0 = const()[name = tensor("input_289_groups_0"), val = tensor(1)]; - tensor input_289 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_289_dilations_0, groups = input_289_groups_0, pad = input_289_pad_0, pad_type = input_289_pad_type_0, strides = input_289_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_287)[name = tensor("input_289")]; - tensor conv_out_35_perm_0 = const()[name = tensor("conv_out_35_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2124_begin_0 = const()[name = tensor("op_2124_begin_0"), val = tensor([0, -1, 0])]; - tensor var_2124_end_0 = const()[name = tensor("op_2124_end_0"), val = tensor([1, 16, 256])]; - tensor var_2124_end_mask_0 = const()[name = tensor("op_2124_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_35 = transpose(perm = conv_out_35_perm_0, x = input_289)[name = tensor("transpose_35")]; - tensor var_2124 = slice_by_index(begin = var_2124_begin_0, end = var_2124_end_0, end_mask = var_2124_end_mask_0, x = conv_out_35)[name = tensor("op_2124")]; - tensor var_2127_begin_0 = const()[name = tensor("op_2127_begin_0"), val = tensor([0, 3, 0])]; - tensor var_2127_end_0 = const()[name = tensor("op_2127_end_0"), val = tensor([1, 4, 256])]; - tensor var_2127_end_mask_0 = const()[name = tensor("op_2127_end_mask_0"), val = tensor([true, false, true])]; - tensor var_2127 = slice_by_index(begin = var_2127_begin_0, end = var_2127_end_0, end_mask = var_2127_end_mask_0, x = x_45)[name = tensor("op_2127")]; - tensor var_2130_begin_0 = const()[name = tensor("op_2130_begin_0"), val = tensor([0, 1, 0])]; - tensor var_2130_end_0 = const()[name = tensor("op_2130_end_0"), val = tensor([1, 16, 256])]; - tensor var_2130_end_mask_0 = const()[name = tensor("op_2130_end_mask_0"), val = tensor([true, true, true])]; - tensor var_2130 = slice_by_index(begin = var_2130_begin_0, end = var_2130_end_0, end_mask = var_2130_end_mask_0, x = input_281)[name = tensor("op_2130")]; + tensor x_expanded_35 = tile(reps = x_expanded_35_reps_0, x = var_1290)[name = tensor("x_expanded_35")]; + tensor var_1304 = sub(x = var_18, y = fill_mask_35)[name = tensor("op_1304")]; + tensor var_1305 = mul(x = window_41, y = var_1304)[name = tensor("op_1305")]; + tensor var_1306 = mul(x = x_expanded_35, y = fill_mask_35)[name = tensor("op_1306")]; + tensor filled_window_35 = add(x = var_1305, y = var_1306)[name = tensor("filled_window_35")]; + tensor var_1308 = greater_equal(x = clip_33, y = var_17)[name = tensor("op_1308")]; + tensor var_1309_dtype_0 = const()[name = tensor("op_1309_dtype_0"), val = tensor("fp32")]; + tensor var_1310 = const()[name = tensor("op_1310"), val = tensor([1, 1, 1])]; + tensor var_1309 = cast(dtype = var_1309_dtype_0, x = var_1308)[name = tensor("cast_156")]; + tensor use_roll_35 = reshape(shape = var_1310, x = var_1309)[name = tensor("use_roll_35")]; + tensor var_1312 = sub(x = var_18, y = use_roll_35)[name = tensor("op_1312")]; + tensor var_1313 = mul(x = filled_window_35, y = var_1312)[name = tensor("op_1313")]; + tensor var_1314 = mul(x = rolled_35, y = use_roll_35)[name = tensor("op_1314")]; + tensor window_43 = add(x = var_1313, y = var_1314)[name = tensor("window_43")]; + tensor var_1316 = const()[name = tensor("op_1316"), val = tensor(0x1p+0)]; + tensor var_1317 = add(x = clip_33, y = var_1316)[name = tensor("op_1317")]; + tensor clip_35 = clip(alpha = var_22, beta = var_17, x = var_1317)[name = tensor("clip_35")]; + tensor var_1320_begin_0 = const()[name = tensor("op_1320_begin_0"), val = tensor([0, 3, 0])]; + tensor var_1320_end_0 = const()[name = tensor("op_1320_end_0"), val = tensor([1, 4, 256])]; + tensor var_1320_end_mask_0 = const()[name = tensor("op_1320_end_mask_0"), val = tensor([true, false, true])]; + tensor var_1320 = slice_by_index(begin = var_1320_begin_0, end = var_1320_end_0, end_mask = var_1320_end_mask_0, x = x_21)[name = tensor("op_1320")]; + tensor var_1323_begin_0 = const()[name = tensor("op_1323_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1323_end_0 = const()[name = tensor("op_1323_end_0"), val = tensor([1, 16, 256])]; + tensor var_1323_end_mask_0 = const()[name = tensor("op_1323_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1323 = slice_by_index(begin = var_1323_begin_0, end = var_1323_end_0, end_mask = var_1323_end_mask_0, x = window_43)[name = tensor("op_1323")]; tensor rolled_37_interleave_0 = const()[name = tensor("rolled_37_interleave_0"), val = tensor(false)]; - tensor rolled_37 = concat(axis = var_34, interleave = rolled_37_interleave_0, values = (var_2130, var_2127))[name = tensor("rolled_37")]; + tensor rolled_37 = concat(axis = var_34, interleave = rolled_37_interleave_0, values = (var_1323, var_1320))[name = tensor("rolled_37")]; tensor clip_36 = clip(alpha = var_22, beta = var_21, x = clip_35)[name = tensor("clip_36")]; - tensor var_2135 = equal(x = encoder__conv_positions, y = clip_36)[name = tensor("op_2135")]; - tensor var_2136_dtype_0 = const()[name = tensor("op_2136_dtype_0"), val = tensor("fp32")]; - tensor var_2137 = const()[name = tensor("op_2137"), val = tensor([1, 16, 1])]; - tensor var_2136 = cast(dtype = var_2136_dtype_0, x = var_2135)[name = tensor("cast_155")]; - tensor fill_mask_37 = reshape(shape = var_2137, x = var_2136)[name = tensor("fill_mask_37")]; + tensor var_1328 = equal(x = encoder__conv_positions, y = clip_36)[name = tensor("op_1328")]; + tensor var_1329_dtype_0 = const()[name = tensor("op_1329_dtype_0"), val = tensor("fp32")]; + tensor var_1330 = const()[name = tensor("op_1330"), val = tensor([1, 16, 1])]; + tensor var_1329 = cast(dtype = var_1329_dtype_0, x = var_1328)[name = tensor("cast_155")]; + tensor fill_mask_37 = reshape(shape = var_1330, x = var_1329)[name = tensor("fill_mask_37")]; tensor x_expanded_37_reps_0 = const()[name = tensor("x_expanded_37_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded_37 = tile(reps = x_expanded_37_reps_0, x = var_2127)[name = tensor("x_expanded_37")]; - tensor var_2141 = sub(x = var_18, y = fill_mask_37)[name = tensor("op_2141")]; - tensor var_2142 = mul(x = input_281, y = var_2141)[name = tensor("op_2142")]; - tensor var_2143 = mul(x = x_expanded_37, y = fill_mask_37)[name = tensor("op_2143")]; - tensor filled_window_37 = add(x = var_2142, y = var_2143)[name = tensor("filled_window_37")]; - tensor var_2145 = greater_equal(x = clip_35, y = var_17)[name = tensor("op_2145")]; - tensor var_2146_dtype_0 = const()[name = tensor("op_2146_dtype_0"), val = tensor("fp32")]; - tensor var_2147 = const()[name = tensor("op_2147"), val = tensor([1, 1, 1])]; - tensor var_2146 = cast(dtype = var_2146_dtype_0, x = var_2145)[name = tensor("cast_154")]; - tensor use_roll_37 = reshape(shape = var_2147, x = var_2146)[name = tensor("use_roll_37")]; - tensor var_2149 = sub(x = var_18, y = use_roll_37)[name = tensor("op_2149")]; - tensor var_2150 = mul(x = filled_window_37, y = var_2149)[name = tensor("op_2150")]; - tensor var_2151 = mul(x = rolled_37, y = use_roll_37)[name = tensor("op_2151")]; - tensor input_291 = add(x = var_2150, y = var_2151)[name = tensor("input_291")]; - tensor var_2153 = const()[name = tensor("op_2153"), val = tensor(0x1p+0)]; - tensor var_2154 = add(x = clip_35, y = var_2153)[name = tensor("op_2154")]; - tensor clip_37 = clip(alpha = var_22, beta = var_17, x = var_2154)[name = tensor("clip_37")]; - tensor x_53_axes_0 = const()[name = tensor("x_53_axes_0"), val = tensor([-1])]; - tensor x_53 = layer_norm(axes = x_53_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_291)[name = tensor("x_53")]; - tensor input_293_perm_0 = const()[name = tensor("input_293_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_123_pad_type_0 = const()[name = tensor("inputs_123_pad_type_0"), val = tensor("valid")]; - tensor inputs_123_strides_0 = const()[name = tensor("inputs_123_strides_0"), val = tensor([1])]; - tensor inputs_123_pad_0 = const()[name = tensor("inputs_123_pad_0"), val = tensor([0, 0])]; - tensor inputs_123_dilations_0 = const()[name = tensor("inputs_123_dilations_0"), val = tensor([1])]; - tensor inputs_123_groups_0 = const()[name = tensor("inputs_123_groups_0"), val = tensor(1)]; - tensor input_293 = transpose(perm = input_293_perm_0, x = x_53)[name = tensor("transpose_34")]; - tensor inputs_123 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_123_dilations_0, groups = inputs_123_groups_0, pad = inputs_123_pad_0, pad_type = inputs_123_pad_type_0, strides = inputs_123_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_293)[name = tensor("inputs_123")]; - tensor var_2175_split_sizes_0 = const()[name = tensor("op_2175_split_sizes_0"), val = tensor([256, 256])]; - tensor var_2175_axis_0 = const()[name = tensor("op_2175_axis_0"), val = tensor(1)]; - tensor var_2175_0, tensor var_2175_1 = split(axis = var_2175_axis_0, split_sizes = var_2175_split_sizes_0, x = inputs_123)[name = tensor("op_2175")]; - tensor var_2177 = sigmoid(x = var_2175_1)[name = tensor("op_2177")]; - tensor inputs_125 = mul(x = var_2175_0, y = var_2177)[name = tensor("inputs_125")]; - tensor outputs_aug_37_pad_type_0 = const()[name = tensor("outputs_aug_37_pad_type_0"), val = tensor("custom")]; - tensor outputs_aug_37_pad_0 = const()[name = tensor("outputs_aug_37_pad_0"), val = tensor([15, 15])]; - tensor outputs_aug_37_groups_0 = const()[name = tensor("outputs_aug_37_groups_0"), val = tensor(256)]; - tensor outputs_aug_37_strides_0 = const()[name = tensor("outputs_aug_37_strides_0"), val = tensor([1])]; - tensor outputs_aug_37_dilations_0 = const()[name = tensor("outputs_aug_37_dilations_0"), val = tensor([1])]; - tensor outputs_aug_37 = conv(dilations = outputs_aug_37_dilations_0, groups = outputs_aug_37_groups_0, pad = outputs_aug_37_pad_0, pad_type = outputs_aug_37_pad_type_0, strides = outputs_aug_37_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_125)[name = tensor("outputs_aug_37")]; - tensor input_295_begin_0 = const()[name = tensor("input_295_begin_0"), val = tensor([0, 0, 0])]; - tensor input_295_end_0 = const()[name = tensor("input_295_end_0"), val = tensor([1, 256, 16])]; - tensor input_295_end_mask_0 = const()[name = tensor("input_295_end_mask_0"), val = tensor([true, true, false])]; - tensor input_295 = slice_by_index(begin = input_295_begin_0, end = input_295_end_0, end_mask = input_295_end_mask_0, x = outputs_aug_37)[name = tensor("input_295")]; - tensor inputs_127 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_295)[name = tensor("inputs_127")]; - tensor input_297 = silu(x = inputs_127)[name = tensor("input_297")]; - tensor input_299_pad_type_0 = const()[name = tensor("input_299_pad_type_0"), val = tensor("valid")]; - tensor input_299_strides_0 = const()[name = tensor("input_299_strides_0"), val = tensor([1])]; - tensor input_299_pad_0 = const()[name = tensor("input_299_pad_0"), val = tensor([0, 0])]; - tensor input_299_dilations_0 = const()[name = tensor("input_299_dilations_0"), val = tensor([1])]; - tensor input_299_groups_0 = const()[name = tensor("input_299_groups_0"), val = tensor(1)]; - tensor input_299 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_299_dilations_0, groups = input_299_groups_0, pad = input_299_pad_0, pad_type = input_299_pad_type_0, strides = input_299_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_297)[name = tensor("input_299")]; - tensor conv_out_37_perm_0 = const()[name = tensor("conv_out_37_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2208_begin_0 = const()[name = tensor("op_2208_begin_0"), val = tensor([0, -1, 0])]; - tensor var_2208_end_0 = const()[name = tensor("op_2208_end_0"), val = tensor([1, 16, 256])]; - tensor var_2208_end_mask_0 = const()[name = tensor("op_2208_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_37 = transpose(perm = conv_out_37_perm_0, x = input_299)[name = tensor("transpose_33")]; - tensor var_2208 = slice_by_index(begin = var_2208_begin_0, end = var_2208_end_0, end_mask = var_2208_end_mask_0, x = conv_out_37)[name = tensor("op_2208")]; - tensor var_2211_begin_0 = const()[name = tensor("op_2211_begin_0"), val = tensor([0, 4, 0])]; - tensor var_2211_end_0 = const()[name = tensor("op_2211_end_0"), val = tensor([1, 1, 256])]; - tensor var_2211_end_mask_0 = const()[name = tensor("op_2211_end_mask_0"), val = tensor([true, true, true])]; - tensor var_2211 = slice_by_index(begin = var_2211_begin_0, end = var_2211_end_0, end_mask = var_2211_end_mask_0, x = x_45)[name = tensor("op_2211")]; - tensor var_2214_begin_0 = const()[name = tensor("op_2214_begin_0"), val = tensor([0, 1, 0])]; - tensor var_2214_end_0 = const()[name = tensor("op_2214_end_0"), val = tensor([1, 16, 256])]; - tensor var_2214_end_mask_0 = const()[name = tensor("op_2214_end_mask_0"), val = tensor([true, true, true])]; - tensor var_2214 = slice_by_index(begin = var_2214_begin_0, end = var_2214_end_0, end_mask = var_2214_end_mask_0, x = input_291)[name = tensor("op_2214")]; + tensor x_expanded_37 = tile(reps = x_expanded_37_reps_0, x = var_1320)[name = tensor("x_expanded_37")]; + tensor var_1334 = sub(x = var_18, y = fill_mask_37)[name = tensor("op_1334")]; + tensor var_1335 = mul(x = window_43, y = var_1334)[name = tensor("op_1335")]; + tensor var_1336 = mul(x = x_expanded_37, y = fill_mask_37)[name = tensor("op_1336")]; + tensor filled_window_37 = add(x = var_1335, y = var_1336)[name = tensor("filled_window_37")]; + tensor var_1338 = greater_equal(x = clip_35, y = var_17)[name = tensor("op_1338")]; + tensor var_1339_dtype_0 = const()[name = tensor("op_1339_dtype_0"), val = tensor("fp32")]; + tensor var_1340 = const()[name = tensor("op_1340"), val = tensor([1, 1, 1])]; + tensor var_1339 = cast(dtype = var_1339_dtype_0, x = var_1338)[name = tensor("cast_154")]; + tensor use_roll_37 = reshape(shape = var_1340, x = var_1339)[name = tensor("use_roll_37")]; + tensor var_1342 = sub(x = var_18, y = use_roll_37)[name = tensor("op_1342")]; + tensor var_1343 = mul(x = filled_window_37, y = var_1342)[name = tensor("op_1343")]; + tensor var_1344 = mul(x = rolled_37, y = use_roll_37)[name = tensor("op_1344")]; + tensor window_45 = add(x = var_1343, y = var_1344)[name = tensor("window_45")]; + tensor var_1346 = const()[name = tensor("op_1346"), val = tensor(0x1p+0)]; + tensor var_1347 = add(x = clip_35, y = var_1346)[name = tensor("op_1347")]; + tensor clip_37 = clip(alpha = var_22, beta = var_17, x = var_1347)[name = tensor("clip_37")]; + tensor var_1350_begin_0 = const()[name = tensor("op_1350_begin_0"), val = tensor([0, 4, 0])]; + tensor var_1350_end_0 = const()[name = tensor("op_1350_end_0"), val = tensor([1, 1, 256])]; + tensor var_1350_end_mask_0 = const()[name = tensor("op_1350_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1350 = slice_by_index(begin = var_1350_begin_0, end = var_1350_end_0, end_mask = var_1350_end_mask_0, x = x_21)[name = tensor("op_1350")]; + tensor var_1353_begin_0 = const()[name = tensor("op_1353_begin_0"), val = tensor([0, 1, 0])]; + tensor var_1353_end_0 = const()[name = tensor("op_1353_end_0"), val = tensor([1, 16, 256])]; + tensor var_1353_end_mask_0 = const()[name = tensor("op_1353_end_mask_0"), val = tensor([true, true, true])]; + tensor var_1353 = slice_by_index(begin = var_1353_begin_0, end = var_1353_end_0, end_mask = var_1353_end_mask_0, x = window_45)[name = tensor("op_1353")]; tensor rolled_interleave_0 = const()[name = tensor("rolled_interleave_0"), val = tensor(false)]; - tensor rolled = concat(axis = var_34, interleave = rolled_interleave_0, values = (var_2214, var_2211))[name = tensor("rolled")]; + tensor rolled = concat(axis = var_34, interleave = rolled_interleave_0, values = (var_1353, var_1350))[name = tensor("rolled")]; tensor clip_38 = clip(alpha = var_22, beta = var_21, x = clip_37)[name = tensor("clip_38")]; - tensor var_2219 = equal(x = encoder__conv_positions, y = clip_38)[name = tensor("op_2219")]; - tensor var_2220_dtype_0 = const()[name = tensor("op_2220_dtype_0"), val = tensor("fp32")]; - tensor var_2221 = const()[name = tensor("op_2221"), val = tensor([1, 16, 1])]; - tensor var_2220 = cast(dtype = var_2220_dtype_0, x = var_2219)[name = tensor("cast_153")]; - tensor fill_mask = reshape(shape = var_2221, x = var_2220)[name = tensor("fill_mask")]; + tensor var_1358 = equal(x = encoder__conv_positions, y = clip_38)[name = tensor("op_1358")]; + tensor var_1359_dtype_0 = const()[name = tensor("op_1359_dtype_0"), val = tensor("fp32")]; + tensor var_1360 = const()[name = tensor("op_1360"), val = tensor([1, 16, 1])]; + tensor var_1359 = cast(dtype = var_1359_dtype_0, x = var_1358)[name = tensor("cast_153")]; + tensor fill_mask = reshape(shape = var_1360, x = var_1359)[name = tensor("fill_mask")]; tensor x_expanded_reps_0 = const()[name = tensor("x_expanded_reps_0"), val = tensor([1, 16, 1])]; - tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_2211)[name = tensor("x_expanded")]; - tensor var_2225 = sub(x = var_18, y = fill_mask)[name = tensor("op_2225")]; - tensor var_2226 = mul(x = input_291, y = var_2225)[name = tensor("op_2226")]; - tensor var_2227 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_2227")]; - tensor filled_window = add(x = var_2226, y = var_2227)[name = tensor("filled_window")]; - tensor var_2229 = greater_equal(x = clip_37, y = var_17)[name = tensor("op_2229")]; - tensor var_2230_dtype_0 = const()[name = tensor("op_2230_dtype_0"), val = tensor("fp32")]; - tensor var_2231 = const()[name = tensor("op_2231"), val = tensor([1, 1, 1])]; - tensor var_2230 = cast(dtype = var_2230_dtype_0, x = var_2229)[name = tensor("cast_152")]; - tensor use_roll = reshape(shape = var_2231, x = var_2230)[name = tensor("use_roll")]; - tensor var_2233 = sub(x = var_18, y = use_roll)[name = tensor("op_2233")]; - tensor var_2234 = mul(x = filled_window, y = var_2233)[name = tensor("op_2234")]; - tensor var_2235 = mul(x = rolled, y = use_roll)[name = tensor("op_2235")]; - tensor input_301 = add(x = var_2234, y = var_2235)[name = tensor("input_301")]; - tensor var_2237 = const()[name = tensor("op_2237"), val = tensor(0x1p+0)]; - tensor var_2238 = add(x = clip_37, y = var_2237)[name = tensor("op_2238")]; - tensor clip_39 = clip(alpha = var_22, beta = var_17, x = var_2238)[name = tensor("clip_39")]; - tensor x_55_axes_0 = const()[name = tensor("x_55_axes_0"), val = tensor([-1])]; - tensor x_55 = layer_norm(axes = x_55_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_301)[name = tensor("x_55")]; - tensor input_303_perm_0 = const()[name = tensor("input_303_perm_0"), val = tensor([0, 2, 1])]; - tensor inputs_129_pad_type_0 = const()[name = tensor("inputs_129_pad_type_0"), val = tensor("valid")]; - tensor inputs_129_strides_0 = const()[name = tensor("inputs_129_strides_0"), val = tensor([1])]; - tensor inputs_129_pad_0 = const()[name = tensor("inputs_129_pad_0"), val = tensor([0, 0])]; - tensor inputs_129_dilations_0 = const()[name = tensor("inputs_129_dilations_0"), val = tensor([1])]; - tensor inputs_129_groups_0 = const()[name = tensor("inputs_129_groups_0"), val = tensor(1)]; - tensor input_303 = transpose(perm = input_303_perm_0, x = x_55)[name = tensor("transpose_32")]; - tensor inputs_129 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_129_dilations_0, groups = inputs_129_groups_0, pad = inputs_129_pad_0, pad_type = inputs_129_pad_type_0, strides = inputs_129_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_303)[name = tensor("inputs_129")]; - tensor var_2259_split_sizes_0 = const()[name = tensor("op_2259_split_sizes_0"), val = tensor([256, 256])]; - tensor var_2259_axis_0 = const()[name = tensor("op_2259_axis_0"), val = tensor(1)]; - tensor var_2259_0, tensor var_2259_1 = split(axis = var_2259_axis_0, split_sizes = var_2259_split_sizes_0, x = inputs_129)[name = tensor("op_2259")]; - tensor var_2261 = sigmoid(x = var_2259_1)[name = tensor("op_2261")]; - tensor inputs_131 = mul(x = var_2259_0, y = var_2261)[name = tensor("inputs_131")]; + tensor x_expanded = tile(reps = x_expanded_reps_0, x = var_1350)[name = tensor("x_expanded")]; + tensor var_1364 = sub(x = var_18, y = fill_mask)[name = tensor("op_1364")]; + tensor var_1365 = mul(x = window_45, y = var_1364)[name = tensor("op_1365")]; + tensor var_1366 = mul(x = x_expanded, y = fill_mask)[name = tensor("op_1366")]; + tensor filled_window = add(x = var_1365, y = var_1366)[name = tensor("filled_window")]; + tensor var_1368 = greater_equal(x = clip_37, y = var_17)[name = tensor("op_1368")]; + tensor var_1369_dtype_0 = const()[name = tensor("op_1369_dtype_0"), val = tensor("fp32")]; + tensor var_1370 = const()[name = tensor("op_1370"), val = tensor([1, 1, 1])]; + tensor var_1369 = cast(dtype = var_1369_dtype_0, x = var_1368)[name = tensor("cast_152")]; + tensor use_roll = reshape(shape = var_1370, x = var_1369)[name = tensor("use_roll")]; + tensor var_1372 = sub(x = var_18, y = use_roll)[name = tensor("op_1372")]; + tensor var_1373 = mul(x = filled_window, y = var_1372)[name = tensor("op_1373")]; + tensor var_1374 = mul(x = rolled, y = use_roll)[name = tensor("op_1374")]; + tensor window = add(x = var_1373, y = var_1374)[name = tensor("window")]; + tensor var_1376 = const()[name = tensor("op_1376"), val = tensor(0x1p+0)]; + tensor var_1377 = add(x = clip_37, y = var_1376)[name = tensor("op_1377")]; + tensor clip_39 = clip(alpha = var_22, beta = var_17, x = var_1377)[name = tensor("clip_39")]; + tensor input_141_interleave_0 = const()[name = tensor("input_141_interleave_0"), val = tensor(false)]; + tensor input_141 = concat(axis = var_31, interleave = input_141_interleave_0, values = (window_39, window_41, window_43, window_45, window))[name = tensor("input_141")]; + tensor x_23_axes_0 = const()[name = tensor("x_23_axes_0"), val = tensor([-1])]; + tensor x_23 = layer_norm(axes = x_23_axes_0, beta = encoder_conv_module_3_sequential_0_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_0_weight, x = input_141)[name = tensor("x_23")]; + tensor input_143_perm_0 = const()[name = tensor("input_143_perm_0"), val = tensor([0, 2, 1])]; + tensor inputs_33_pad_type_0 = const()[name = tensor("inputs_33_pad_type_0"), val = tensor("valid")]; + tensor inputs_33_strides_0 = const()[name = tensor("inputs_33_strides_0"), val = tensor([1])]; + tensor inputs_33_pad_0 = const()[name = tensor("inputs_33_pad_0"), val = tensor([0, 0])]; + tensor inputs_33_dilations_0 = const()[name = tensor("inputs_33_dilations_0"), val = tensor([1])]; + tensor inputs_33_groups_0 = const()[name = tensor("inputs_33_groups_0"), val = tensor(1)]; + tensor input_143 = transpose(perm = input_143_perm_0, x = x_23)[name = tensor("transpose_33")]; + tensor inputs_33 = conv(bias = encoder_conv_module_3_sequential_2_conv_bias, dilations = inputs_33_dilations_0, groups = inputs_33_groups_0, pad = inputs_33_pad_0, pad_type = inputs_33_pad_type_0, strides = inputs_33_strides_0, weight = encoder_conv_module_3_sequential_2_conv_weight, x = input_143)[name = tensor("inputs_33")]; + tensor var_1400_split_sizes_0 = const()[name = tensor("op_1400_split_sizes_0"), val = tensor([256, 256])]; + tensor var_1400_axis_0 = const()[name = tensor("op_1400_axis_0"), val = tensor(1)]; + tensor var_1400_0, tensor var_1400_1 = split(axis = var_1400_axis_0, split_sizes = var_1400_split_sizes_0, x = inputs_33)[name = tensor("op_1400")]; + tensor var_1402 = sigmoid(x = var_1400_1)[name = tensor("op_1402")]; + tensor inputs_35 = mul(x = var_1400_0, y = var_1402)[name = tensor("inputs_35")]; tensor outputs_aug_pad_type_0 = const()[name = tensor("outputs_aug_pad_type_0"), val = tensor("custom")]; tensor outputs_aug_pad_0 = const()[name = tensor("outputs_aug_pad_0"), val = tensor([15, 15])]; tensor outputs_aug_groups_0 = const()[name = tensor("outputs_aug_groups_0"), val = tensor(256)]; tensor outputs_aug_strides_0 = const()[name = tensor("outputs_aug_strides_0"), val = tensor([1])]; tensor outputs_aug_dilations_0 = const()[name = tensor("outputs_aug_dilations_0"), val = tensor([1])]; - tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_131)[name = tensor("outputs_aug")]; - tensor input_305_begin_0 = const()[name = tensor("input_305_begin_0"), val = tensor([0, 0, 0])]; - tensor input_305_end_0 = const()[name = tensor("input_305_end_0"), val = tensor([1, 256, 16])]; - tensor input_305_end_mask_0 = const()[name = tensor("input_305_end_mask_0"), val = tensor([true, true, false])]; - tensor input_305 = slice_by_index(begin = input_305_begin_0, end = input_305_end_0, end_mask = input_305_end_mask_0, x = outputs_aug)[name = tensor("input_305")]; - tensor inputs_133 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_305)[name = tensor("inputs_133")]; - tensor input_307 = silu(x = inputs_133)[name = tensor("input_307")]; - tensor input_309_pad_type_0 = const()[name = tensor("input_309_pad_type_0"), val = tensor("valid")]; - tensor input_309_strides_0 = const()[name = tensor("input_309_strides_0"), val = tensor([1])]; - tensor input_309_pad_0 = const()[name = tensor("input_309_pad_0"), val = tensor([0, 0])]; - tensor input_309_dilations_0 = const()[name = tensor("input_309_dilations_0"), val = tensor([1])]; - tensor input_309_groups_0 = const()[name = tensor("input_309_groups_0"), val = tensor(1)]; - tensor input_309 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_309_dilations_0, groups = input_309_groups_0, pad = input_309_pad_0, pad_type = input_309_pad_type_0, strides = input_309_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_307)[name = tensor("input_309")]; - tensor conv_out_39_perm_0 = const()[name = tensor("conv_out_39_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2292_begin_0 = const()[name = tensor("op_2292_begin_0"), val = tensor([0, -1, 0])]; - tensor var_2292_end_0 = const()[name = tensor("op_2292_end_0"), val = tensor([1, 16, 256])]; - tensor var_2292_end_mask_0 = const()[name = tensor("op_2292_end_mask_0"), val = tensor([true, true, true])]; - tensor conv_out_39 = transpose(perm = conv_out_39_perm_0, x = input_309)[name = tensor("transpose_31")]; - tensor var_2292 = slice_by_index(begin = var_2292_begin_0, end = var_2292_end_0, end_mask = var_2292_end_mask_0, x = conv_out_39)[name = tensor("op_2292")]; - tensor var_2295_interleave_0 = const()[name = tensor("op_2295_interleave_0"), val = tensor(false)]; - tensor var_2295 = concat(axis = var_34, interleave = var_2295_interleave_0, values = (var_1956, var_2040, var_2124, var_2208, var_2292))[name = tensor("op_2295")]; - tensor input_311 = add(x = x_45, y = var_2295)[name = tensor("input_311")]; - tensor input_313_axes_0 = const()[name = tensor("input_313_axes_0"), val = tensor([-1])]; - tensor input_313 = layer_norm(axes = input_313_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_311)[name = tensor("input_313")]; - tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_313)[name = tensor("linear_35")]; - tensor input_315 = silu(x = inputs)[name = tensor("input_315")]; - tensor input_319 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_315)[name = tensor("linear_36")]; - tensor var_2318 = const()[name = tensor("op_2318"), val = tensor(0x1p-1)]; - tensor var_2319 = mul(x = input_319, y = var_2318)[name = tensor("op_2319")]; - tensor input_321 = add(x = var_2319, y = input_311)[name = tensor("input_321")]; - tensor x_57_axes_0 = const()[name = tensor("x_57_axes_0"), val = tensor([-1])]; - tensor x_57 = layer_norm(axes = x_57_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_36, gamma = encoder_layer_norm_3_weight, x = input_321)[name = tensor("x_57")]; + tensor outputs_aug = conv(dilations = outputs_aug_dilations_0, groups = outputs_aug_groups_0, pad = outputs_aug_pad_0, pad_type = outputs_aug_pad_type_0, strides = outputs_aug_strides_0, weight = encoder_conv_module_3_sequential_4_conv_weight, x = inputs_35)[name = tensor("outputs_aug")]; + tensor input_145_begin_0 = const()[name = tensor("input_145_begin_0"), val = tensor([0, 0, 0])]; + tensor input_145_end_0 = const()[name = tensor("input_145_end_0"), val = tensor([5, 256, 16])]; + tensor input_145_end_mask_0 = const()[name = tensor("input_145_end_mask_0"), val = tensor([true, true, false])]; + tensor input_145 = slice_by_index(begin = input_145_begin_0, end = input_145_end_0, end_mask = input_145_end_mask_0, x = outputs_aug)[name = tensor("input_145")]; + tensor inputs_37 = batch_norm(beta = encoder_conv_module_3_sequential_5_bias, epsilon = var_36, gamma = encoder_conv_module_3_sequential_5_weight, mean = encoder_conv_module_3_sequential_5_running_mean, variance = encoder_conv_module_3_sequential_5_running_var, x = input_145)[name = tensor("inputs_37")]; + tensor input_147 = silu(x = inputs_37)[name = tensor("input_147")]; + tensor input_149_pad_type_0 = const()[name = tensor("input_149_pad_type_0"), val = tensor("valid")]; + tensor input_149_strides_0 = const()[name = tensor("input_149_strides_0"), val = tensor([1])]; + tensor input_149_pad_0 = const()[name = tensor("input_149_pad_0"), val = tensor([0, 0])]; + tensor input_149_dilations_0 = const()[name = tensor("input_149_dilations_0"), val = tensor([1])]; + tensor input_149_groups_0 = const()[name = tensor("input_149_groups_0"), val = tensor(1)]; + tensor input_149 = conv(bias = encoder_conv_module_3_sequential_7_conv_bias, dilations = input_149_dilations_0, groups = input_149_groups_0, pad = input_149_pad_0, pad_type = input_149_pad_type_0, strides = input_149_strides_0, weight = encoder_conv_module_3_sequential_7_conv_weight, x = input_147)[name = tensor("input_149")]; + tensor conv_out_7_perm_0 = const()[name = tensor("conv_out_7_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1433_begin_0 = const()[name = tensor("op_1433_begin_0"), val = tensor([0, -1, 0])]; + tensor var_1433_end_0 = const()[name = tensor("op_1433_end_0"), val = tensor([5, 16, 256])]; + tensor var_1433_end_mask_0 = const()[name = tensor("op_1433_end_mask_0"), val = tensor([true, true, true])]; + tensor conv_out_7 = transpose(perm = conv_out_7_perm_0, x = input_149)[name = tensor("transpose_32")]; + tensor var_1433 = slice_by_index(begin = var_1433_begin_0, end = var_1433_end_0, end_mask = var_1433_end_mask_0, x = conv_out_7)[name = tensor("op_1433")]; + tensor var_1435_perm_0 = const()[name = tensor("op_1435_perm_0"), val = tensor([1, 0, 2])]; + tensor var_1435 = transpose(perm = var_1435_perm_0, x = var_1433)[name = tensor("transpose_31")]; + tensor input_151 = add(x = x_21, y = var_1435)[name = tensor("input_151")]; + tensor input_153_axes_0 = const()[name = tensor("input_153_axes_0"), val = tensor([-1])]; + tensor input_153 = layer_norm(axes = input_153_axes_0, beta = encoder_ffn2_3_module_sequential_0_bias, epsilon = var_36, gamma = encoder_ffn2_3_module_sequential_0_weight, x = input_151)[name = tensor("input_153")]; + tensor inputs = linear(bias = encoder_ffn2_3_module_sequential_1_linear_bias, weight = encoder_ffn2_3_module_sequential_1_linear_weight, x = input_153)[name = tensor("linear_35")]; + tensor input_155 = silu(x = inputs)[name = tensor("input_155")]; + tensor input_159 = linear(bias = encoder_ffn2_3_module_sequential_4_linear_bias, weight = encoder_ffn2_3_module_sequential_4_linear_weight, x = input_155)[name = tensor("linear_36")]; + tensor var_1458 = const()[name = tensor("op_1458"), val = tensor(0x1p-1)]; + tensor var_1459 = mul(x = input_159, y = var_1458)[name = tensor("op_1459")]; + tensor input_161 = add(x = var_1459, y = input_151)[name = tensor("input_161")]; + tensor x_25_axes_0 = const()[name = tensor("x_25_axes_0"), val = tensor([-1])]; + tensor x_25 = layer_norm(axes = x_25_axes_0, beta = encoder_layer_norm_3_bias, epsilon = var_36, gamma = encoder_layer_norm_3_weight, x = input_161)[name = tensor("x_25")]; tensor x_ct_perm_0 = const()[name = tensor("x_ct_perm_0"), val = tensor([0, 2, 1])]; tensor cat_interleave_0 = const()[name = tensor("cat_interleave_0"), val = tensor(false)]; - tensor x_ct = transpose(perm = x_ct_perm_0, x = x_57)[name = tensor("transpose_30")]; + tensor x_ct = transpose(perm = x_ct_perm_0, x = x_25)[name = tensor("transpose_30")]; tensor cat = concat(axis = var_28, interleave = cat_interleave_0, values = (cnn_window, x_ct))[name = tensor("cat")]; tensor conv_out_pad_type_0 = const()[name = tensor("conv_out_pad_type_0"), val = tensor("valid")]; tensor conv_out_strides_0 = const()[name = tensor("conv_out_strides_0"), val = tensor([1])]; @@ -2097,44 +1482,44 @@ program(1.0) tensor conv_out_dilations_0 = const()[name = tensor("conv_out_dilations_0"), val = tensor([1])]; tensor conv_out_groups_0 = const()[name = tensor("conv_out_groups_0"), val = tensor(1)]; tensor conv_out = conv(bias = encoder_cnn_bias, dilations = conv_out_dilations_0, groups = conv_out_groups_0, pad = conv_out_pad_0, pad_type = conv_out_pad_type_0, strides = conv_out_strides_0, weight = encoder_cnn_weight, x = cat)[name = tensor("conv_out")]; - tensor var_2337_begin_0 = const()[name = tensor("op_2337_begin_0"), val = tensor([0, 0, 5])]; - tensor var_2337_end_0 = const()[name = tensor("op_2337_end_0"), val = tensor([1, 256, 23])]; - tensor var_2337_end_mask_0 = const()[name = tensor("op_2337_end_mask_0"), val = tensor([true, true, true])]; - tensor cnn_window_new = slice_by_index(begin = var_2337_begin_0, end = var_2337_end_0, end_mask = var_2337_end_mask_0, x = cat)[name = tensor("op_2337")]; - tensor input_323_perm_0 = const()[name = tensor("input_323_perm_0"), val = tensor([0, 2, 1])]; - tensor var_2339 = const()[name = tensor("op_2339"), val = tensor([-1])]; - tensor input_323 = transpose(perm = input_323_perm_0, x = conv_out)[name = tensor("transpose_29")]; - tensor var_2340 = reduce_l2_norm(axes = var_2339, keep_dims = var_37, x = input_323)[name = tensor("op_2340")]; - tensor const_28 = const()[name = tensor("const_28"), val = tensor(0x1.fffffep+127)]; - tensor clip_40 = clip(alpha = var_12, beta = const_28, x = var_2340)[name = tensor("clip_40")]; - tensor emb = real_div(x = input_323, y = clip_40)[name = tensor("emb")]; - tensor var_2344_axis_0 = const()[name = tensor("op_2344_axis_0"), val = tensor(0)]; - tensor enc_kv_new = stack(axis = var_2344_axis_0, values = (var_215, var_763, var_1311, nkv_1))[name = tensor("op_2344")]; - tensor var_2346_axis_0 = const()[name = tensor("op_2346_axis_0"), val = tensor(0)]; - tensor enc_scale_new = stack(axis = var_2346_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_2346")]; - tensor var_2348_axis_0 = const()[name = tensor("op_2348_axis_0"), val = tensor(0)]; - tensor enc_conv_window_new = stack(axis = var_2348_axis_0, values = (input_61, input_141, input_221, input_301))[name = tensor("op_2348")]; - tensor var_2350_axis_0 = const()[name = tensor("op_2350_axis_0"), val = tensor(0)]; - tensor enc_conv_filled_new = stack(axis = var_2350_axis_0, values = (clip_9, clip_19, clip_29, clip_39))[name = tensor("op_2350")]; - tensor var_2360 = const()[name = tensor("op_2360"), val = tensor(0x1.5798eep-27)]; - tensor var_2366 = const()[name = tensor("op_2366"), val = tensor(0x1.4f8b58p-17)]; - tensor var_2368 = const()[name = tensor("op_2368"), val = tensor(0x1.0c6f7ap-20)]; - tensor var_2369 = const()[name = tensor("op_2369"), val = tensor(true)]; - tensor var_2371 = const()[name = tensor("op_2371"), val = tensor(0x1p+0)]; - tensor var_2375 = const()[name = tensor("op_2375"), val = tensor(-1)]; - tensor var_2380 = const()[name = tensor("op_2380"), val = tensor(0)]; + tensor var_1477_begin_0 = const()[name = tensor("op_1477_begin_0"), val = tensor([0, 0, 5])]; + tensor var_1477_end_0 = const()[name = tensor("op_1477_end_0"), val = tensor([1, 256, 23])]; + tensor var_1477_end_mask_0 = const()[name = tensor("op_1477_end_mask_0"), val = tensor([true, true, true])]; + tensor cnn_window_new = slice_by_index(begin = var_1477_begin_0, end = var_1477_end_0, end_mask = var_1477_end_mask_0, x = cat)[name = tensor("op_1477")]; + tensor input_163_perm_0 = const()[name = tensor("input_163_perm_0"), val = tensor([0, 2, 1])]; + tensor var_1479 = const()[name = tensor("op_1479"), val = tensor([-1])]; + tensor input_163 = transpose(perm = input_163_perm_0, x = conv_out)[name = tensor("transpose_29")]; + tensor var_1480 = reduce_l2_norm(axes = var_1479, keep_dims = var_37, x = input_163)[name = tensor("op_1480")]; + tensor const_12 = const()[name = tensor("const_12"), val = tensor(0x1.fffffep+127)]; + tensor clip_40 = clip(alpha = var_12, beta = const_12, x = var_1480)[name = tensor("clip_40")]; + tensor emb = real_div(x = input_163, y = clip_40)[name = tensor("emb")]; + tensor var_1484_axis_0 = const()[name = tensor("op_1484_axis_0"), val = tensor(0)]; + tensor enc_kv_new = stack(axis = var_1484_axis_0, values = (var_215, var_548, var_881, nkv_1))[name = tensor("op_1484")]; + tensor var_1486_axis_0 = const()[name = tensor("op_1486_axis_0"), val = tensor(0)]; + tensor enc_scale_new = stack(axis = var_1486_axis_0, values = (new_scale_1, new_scale_3, new_scale_5, new_scale_7))[name = tensor("op_1486")]; + tensor var_1488_axis_0 = const()[name = tensor("op_1488_axis_0"), val = tensor(0)]; + tensor enc_conv_window_new = stack(axis = var_1488_axis_0, values = (window_11, window_23, window_35, window))[name = tensor("op_1488")]; + tensor var_1490_axis_0 = const()[name = tensor("op_1490_axis_0"), val = tensor(0)]; + tensor enc_conv_filled_new = stack(axis = var_1490_axis_0, values = (clip_9, clip_19, clip_29, clip_39))[name = tensor("op_1490")]; + tensor var_1500 = const()[name = tensor("op_1500"), val = tensor(0x1.5798eep-27)]; + tensor var_1505 = const()[name = tensor("op_1505"), val = tensor(0x1.4f8b58p-17)]; + tensor var_1507 = const()[name = tensor("op_1507"), val = tensor(0x1.0c6f7ap-20)]; + tensor var_1508 = const()[name = tensor("op_1508"), val = tensor(true)]; + tensor var_1510 = const()[name = tensor("op_1510"), val = tensor(0x1p+0)]; + tensor var_1514 = const()[name = tensor("op_1514"), val = tensor(-1)]; + tensor var_1520 = const()[name = tensor("op_1520"), val = tensor(0)]; tensor pos = const()[name = tensor("pos"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(44395840)))]; - tensor var_2442_axes_0 = const()[name = tensor("op_2442_axes_0"), val = tensor([2])]; - tensor var_2442 = expand_dims(axes = var_2442_axes_0, x = emb)[name = tensor("op_2442")]; - tensor var_2443 = const()[name = tensor("op_2443"), val = tensor([1, 1, 12, 1])]; - tensor emb_exp = tile(reps = var_2443, x = var_2442)[name = tensor("emb_exp")]; - tensor input_325_interleave_0 = const()[name = tensor("input_325_interleave_0"), val = tensor(false)]; - tensor input_325 = concat(axis = var_2375, interleave = input_325_interleave_0, values = (emb_exp, pos))[name = tensor("input_325")]; - tensor x_59 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_325)[name = tensor("linear_37")]; - tensor var_2450_perm_0 = const()[name = tensor("op_2450_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2454 = const()[name = tensor("op_2454"), val = tensor([12, 5, 256])]; - tensor var_2450 = transpose(perm = var_2450_perm_0, x = x_59)[name = tensor("transpose_28")]; - tensor x_61 = reshape(shape = var_2454, x = var_2450)[name = tensor("x_61")]; + tensor var_1582_axes_0 = const()[name = tensor("op_1582_axes_0"), val = tensor([2])]; + tensor var_1582 = expand_dims(axes = var_1582_axes_0, x = emb)[name = tensor("op_1582")]; + tensor emb_exp_reps_0 = const()[name = tensor("emb_exp_reps_0"), val = tensor([1, 1, 12, 1])]; + tensor emb_exp = tile(reps = emb_exp_reps_0, x = var_1582)[name = tensor("emb_exp")]; + tensor input_165_interleave_0 = const()[name = tensor("input_165_interleave_0"), val = tensor(false)]; + tensor input_165 = concat(axis = var_1514, interleave = input_165_interleave_0, values = (emb_exp, pos))[name = tensor("input_165")]; + tensor x_27 = linear(bias = decoder_convert_bias, weight = decoder_convert_weight, x = input_165)[name = tensor("linear_37")]; + tensor var_1590_perm_0 = const()[name = tensor("op_1590_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1594 = const()[name = tensor("op_1594"), val = tensor([12, 5, 256])]; + tensor var_1590 = transpose(perm = var_1590_perm_0, x = x_27)[name = tensor("transpose_28")]; + tensor x_29 = reshape(shape = var_1594, x = var_1590)[name = tensor("x_29")]; tensor prev_kv_9_begin_0 = const()[name = tensor("prev_kv_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; tensor prev_kv_9_end_0 = const()[name = tensor("prev_kv_9_end_0"), val = tensor([1, 12, 4, 64, 64])]; tensor prev_kv_9_end_mask_0 = const()[name = tensor("prev_kv_9_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -2145,132 +1530,132 @@ program(1.0) tensor prev_scale_9_end_mask_0 = const()[name = tensor("prev_scale_9_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_9_squeeze_mask_0 = const()[name = tensor("prev_scale_9_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale_9 = slice_by_index(begin = prev_scale_9_begin_0, end = prev_scale_9_end_0, end_mask = prev_scale_9_end_mask_0, squeeze_mask = prev_scale_9_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale_9")]; - tensor var_2462 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_61)[name = tensor("linear_38")]; - tensor var_2463 = const()[name = tensor("op_2463"), val = tensor([12, 5, 4, 64])]; - tensor var_2464 = reshape(shape = var_2463, x = var_2462)[name = tensor("op_2464")]; + tensor var_1602 = linear(bias = decoder_q_proj_0_bias, weight = decoder_q_proj_0_weight, x = x_29)[name = tensor("linear_38")]; + tensor var_1603 = const()[name = tensor("op_1603"), val = tensor([12, 5, 4, 64])]; + tensor var_1604 = reshape(shape = var_1603, x = var_1602)[name = tensor("op_1604")]; tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2468 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_61)[name = tensor("linear_39")]; - tensor var_2469 = const()[name = tensor("op_2469"), val = tensor(0x1p-3)]; - tensor var_2470 = mul(x = var_2468, y = var_2469)[name = tensor("op_2470")]; - tensor var_2471 = const()[name = tensor("op_2471"), val = tensor([12, 5, 4, 64])]; - tensor var_2472 = reshape(shape = var_2471, x = var_2470)[name = tensor("op_2472")]; + tensor var_1608 = linear(bias = decoder_k_proj_0_bias, weight = decoder_k_proj_0_weight, x = x_29)[name = tensor("linear_39")]; + tensor var_1609 = const()[name = tensor("op_1609"), val = tensor(0x1p-3)]; + tensor var_1610 = mul(x = var_1608, y = var_1609)[name = tensor("op_1610")]; + tensor var_1611 = const()[name = tensor("op_1611"), val = tensor([12, 5, 4, 64])]; + tensor var_1612 = reshape(shape = var_1611, x = var_1610)[name = tensor("op_1612")]; tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2476 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_61)[name = tensor("linear_40")]; - tensor var_2477 = const()[name = tensor("op_2477"), val = tensor([12, 5, 4, 64])]; - tensor var_2478 = reshape(shape = var_2477, x = var_2476)[name = tensor("op_2478")]; + tensor var_1616 = linear(bias = decoder_v_proj_0_bias, weight = decoder_v_proj_0_weight, x = x_29)[name = tensor("linear_40")]; + tensor var_1617 = const()[name = tensor("op_1617"), val = tensor([12, 5, 4, 64])]; + tensor var_1618 = reshape(shape = var_1617, x = var_1616)[name = tensor("op_1618")]; tensor v_9_perm_0 = const()[name = tensor("v_9_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_329 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_61)[name = tensor("linear_41")]; + tensor input_169 = linear(bias = decoder_g_proj_0_bias, weight = decoder_g_proj_0_weight, x = x_29)[name = tensor("linear_41")]; tensor cumsum_mask_1_exclusive_0 = const()[name = tensor("cumsum_mask_1_exclusive_0"), val = tensor(false)]; tensor cumsum_mask_1_reverse_0 = const()[name = tensor("cumsum_mask_1_reverse_0"), val = tensor(false)]; - tensor cumsum_mask_1 = cumsum(axis = var_2380, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; + tensor cumsum_mask_1 = cumsum(axis = var_1520, exclusive = cumsum_mask_1_exclusive_0, reverse = cumsum_mask_1_reverse_0, x = valid_mask)[name = tensor("cumsum_mask_1")]; tensor sqrt_s0_9 = sqrt(x = prev_scale_9)[name = tensor("sqrt_s0_9")]; tensor s_t_9 = add(x = prev_scale_9, y = cumsum_mask_1)[name = tensor("s_t_9")]; - tensor const_36 = const()[name = tensor("const_36"), val = tensor(0x1.fffffep+127)]; - tensor clip_41 = clip(alpha = var_2371, beta = const_36, x = s_t_9)[name = tensor("clip_41")]; + tensor const_20 = const()[name = tensor("const_20"), val = tensor(0x1.fffffep+127)]; + tensor clip_41 = clip(alpha = var_1510, beta = const_20, x = s_t_9)[name = tensor("clip_41")]; tensor sqrt_s_t_9 = sqrt(x = clip_41)[name = tensor("sqrt_s_t_9")]; tensor qk_9_transpose_x_1 = const()[name = tensor("qk_9_transpose_x_1"), val = tensor(false)]; tensor qk_9_transpose_y_1 = const()[name = tensor("qk_9_transpose_y_1"), val = tensor(true)]; - tensor k_9 = transpose(perm = k_9_perm_0, x = var_2472)[name = tensor("transpose_26")]; - tensor q_9 = transpose(perm = q_9_perm_0, x = var_2464)[name = tensor("transpose_27")]; + tensor k_9 = transpose(perm = k_9_perm_0, x = var_1612)[name = tensor("transpose_26")]; + tensor q_9 = transpose(perm = q_9_perm_0, x = var_1604)[name = tensor("transpose_27")]; tensor qk_9 = matmul(transpose_x = qk_9_transpose_x_1, transpose_y = qk_9_transpose_y_1, x = q_9, y = k_9)[name = tensor("qk_9")]; - tensor var_2490 = const()[name = tensor("op_2490"), val = tensor([1, 5])]; - tensor var_2491 = reshape(shape = var_2490, x = valid_mask)[name = tensor("op_2491")]; - tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_2491)[name = tensor("causal_with_valid_1")]; - tensor var_2493 = const()[name = tensor("op_2493"), val = tensor([5, 1])]; - tensor var_2494 = reshape(shape = var_2493, x = sqrt_s_t_9)[name = tensor("op_2494")]; - tensor M_9 = real_div(x = causal_with_valid_1, y = var_2494)[name = tensor("M_9")]; - tensor var_2496 = mul(x = qk_9, y = M_9)[name = tensor("op_2496")]; + tensor var_1630 = const()[name = tensor("op_1630"), val = tensor([1, 5])]; + tensor var_1631 = reshape(shape = var_1630, x = valid_mask)[name = tensor("op_1631")]; + tensor causal_with_valid_1 = mul(x = decoder__causal_mask, y = var_1631)[name = tensor("causal_with_valid_1")]; + tensor var_1633 = const()[name = tensor("op_1633"), val = tensor([5, 1])]; + tensor var_1634 = reshape(shape = var_1633, x = sqrt_s_t_9)[name = tensor("op_1634")]; + tensor M_9 = real_div(x = causal_with_valid_1, y = var_1634)[name = tensor("M_9")]; + tensor var_1636 = mul(x = qk_9, y = M_9)[name = tensor("op_1636")]; tensor inner_9_transpose_x_0 = const()[name = tensor("inner_9_transpose_x_0"), val = tensor(false)]; tensor inner_9_transpose_y_0 = const()[name = tensor("inner_9_transpose_y_0"), val = tensor(false)]; - tensor v_9 = transpose(perm = v_9_perm_0, x = var_2478)[name = tensor("transpose_25")]; - tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_2496, y = v_9)[name = tensor("inner_9")]; - tensor var_2498_transpose_x_0 = const()[name = tensor("op_2498_transpose_x_0"), val = tensor(false)]; - tensor var_2498_transpose_y_0 = const()[name = tensor("op_2498_transpose_y_0"), val = tensor(false)]; - tensor var_2498 = matmul(transpose_x = var_2498_transpose_x_0, transpose_y = var_2498_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_2498")]; - tensor var_2499 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_2499")]; - tensor var_2500 = const()[name = tensor("op_2500"), val = tensor([1, 1, 5, 1])]; - tensor var_2501 = reshape(shape = var_2500, x = var_2499)[name = tensor("op_2501")]; - tensor cross_9 = mul(x = var_2498, y = var_2501)[name = tensor("cross_9")]; + tensor v_9 = transpose(perm = v_9_perm_0, x = var_1618)[name = tensor("transpose_25")]; + tensor inner_9 = matmul(transpose_x = inner_9_transpose_x_0, transpose_y = inner_9_transpose_y_0, x = var_1636, y = v_9)[name = tensor("inner_9")]; + tensor var_1638_transpose_x_0 = const()[name = tensor("op_1638_transpose_x_0"), val = tensor(false)]; + tensor var_1638_transpose_y_0 = const()[name = tensor("op_1638_transpose_y_0"), val = tensor(false)]; + tensor var_1638 = matmul(transpose_x = var_1638_transpose_x_0, transpose_y = var_1638_transpose_y_0, x = q_9, y = prev_kv_9)[name = tensor("op_1638")]; + tensor var_1639 = real_div(x = sqrt_s0_9, y = sqrt_s_t_9)[name = tensor("op_1639")]; + tensor var_1640 = const()[name = tensor("op_1640"), val = tensor([1, 1, 5, 1])]; + tensor var_1641 = reshape(shape = var_1640, x = var_1639)[name = tensor("op_1641")]; + tensor cross_9 = mul(x = var_1638, y = var_1641)[name = tensor("cross_9")]; tensor out_25 = add(x = inner_9, y = cross_9)[name = tensor("out_25")]; - tensor var_2504 = const()[name = tensor("op_2504"), val = tensor([1, 1, 5, 1])]; - tensor var_2505 = reshape(shape = var_2504, x = valid_mask)[name = tensor("op_2505")]; - tensor v_masked_1 = mul(x = v_9, y = var_2505)[name = tensor("v_masked_1")]; - tensor var_2507 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_2507")]; - tensor var_2509_transpose_x_1 = const()[name = tensor("op_2509_transpose_x_1"), val = tensor(true)]; - tensor var_2509_transpose_y_1 = const()[name = tensor("op_2509_transpose_y_1"), val = tensor(false)]; - tensor var_2509 = matmul(transpose_x = var_2509_transpose_x_1, transpose_y = var_2509_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_2509")]; - tensor new_kv_unnorm_9 = add(x = var_2507, y = var_2509)[name = tensor("new_kv_unnorm_9")]; - tensor var_2511_keep_dims_0 = const()[name = tensor("op_2511_keep_dims_0"), val = tensor(false)]; - tensor var_2511 = reduce_sum(keep_dims = var_2511_keep_dims_0, x = valid_mask)[name = tensor("op_2511")]; - tensor var_2512 = const()[name = tensor("op_2512"), val = tensor([1])]; - tensor var_2513 = reshape(shape = var_2512, x = var_2511)[name = tensor("op_2513")]; - tensor new_scale_9 = add(x = prev_scale_9, y = var_2513)[name = tensor("new_scale_9")]; - tensor const_37 = const()[name = tensor("const_37"), val = tensor(0x1.fffffep+127)]; - tensor clip_42 = clip(alpha = var_2371, beta = const_37, x = new_scale_9)[name = tensor("clip_42")]; + tensor var_1644 = const()[name = tensor("op_1644"), val = tensor([1, 1, 5, 1])]; + tensor var_1645 = reshape(shape = var_1644, x = valid_mask)[name = tensor("op_1645")]; + tensor v_masked_1 = mul(x = v_9, y = var_1645)[name = tensor("v_masked_1")]; + tensor var_1647 = mul(x = prev_kv_9, y = sqrt_s0_9)[name = tensor("op_1647")]; + tensor var_1649_transpose_x_1 = const()[name = tensor("op_1649_transpose_x_1"), val = tensor(true)]; + tensor var_1649_transpose_y_1 = const()[name = tensor("op_1649_transpose_y_1"), val = tensor(false)]; + tensor var_1649 = matmul(transpose_x = var_1649_transpose_x_1, transpose_y = var_1649_transpose_y_1, x = k_9, y = v_masked_1)[name = tensor("op_1649")]; + tensor new_kv_unnorm_9 = add(x = var_1647, y = var_1649)[name = tensor("new_kv_unnorm_9")]; + tensor var_1651_keep_dims_0 = const()[name = tensor("op_1651_keep_dims_0"), val = tensor(false)]; + tensor var_1651 = reduce_sum(keep_dims = var_1651_keep_dims_0, x = valid_mask)[name = tensor("op_1651")]; + tensor var_1652 = const()[name = tensor("op_1652"), val = tensor([1])]; + tensor var_1653 = reshape(shape = var_1652, x = var_1651)[name = tensor("op_1653")]; + tensor new_scale_9 = add(x = prev_scale_9, y = var_1653)[name = tensor("new_scale_9")]; + tensor const_21 = const()[name = tensor("const_21"), val = tensor(0x1.fffffep+127)]; + tensor clip_42 = clip(alpha = var_1510, beta = const_21, x = new_scale_9)[name = tensor("clip_42")]; tensor sqrt_new_scale_1 = sqrt(x = clip_42)[name = tensor("sqrt_new_scale_1")]; - tensor var_2517 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_2517")]; - tensor var_2518_perm_0 = const()[name = tensor("op_2518_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1657 = real_div(x = new_kv_unnorm_9, y = sqrt_new_scale_1)[name = tensor("op_1657")]; + tensor var_1658_perm_0 = const()[name = tensor("op_1658_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_27_axes_0 = const()[name = tensor("out_27_axes_0"), val = tensor([-1])]; - tensor var_2518 = transpose(perm = var_2518_perm_0, x = out_25)[name = tensor("transpose_24")]; - tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_2368, x = var_2518)[name = tensor("out_27")]; - tensor var_2522 = const()[name = tensor("op_2522"), val = tensor([12, 5, 256])]; - tensor out_29 = reshape(shape = var_2522, x = out_27)[name = tensor("out_29")]; - tensor var_2524 = silu(x = input_329)[name = tensor("op_2524")]; - tensor input_331 = mul(x = var_2524, y = out_29)[name = tensor("input_331")]; - tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_331)[name = tensor("linear_42")]; - tensor input_333 = add(x = x_61, y = ret_out_9)[name = tensor("input_333")]; + tensor var_1658 = transpose(perm = var_1658_perm_0, x = out_25)[name = tensor("transpose_24")]; + tensor out_27 = layer_norm(axes = out_27_axes_0, epsilon = var_1507, x = var_1658)[name = tensor("out_27")]; + tensor var_1662 = const()[name = tensor("op_1662"), val = tensor([12, 5, 256])]; + tensor out_29 = reshape(shape = var_1662, x = out_27)[name = tensor("out_29")]; + tensor var_1664 = silu(x = input_169)[name = tensor("op_1664")]; + tensor input_171 = mul(x = var_1664, y = out_29)[name = tensor("input_171")]; + tensor ret_out_9 = linear(bias = decoder_out_proj_0_bias, weight = decoder_out_proj_0_weight, x = input_171)[name = tensor("linear_42")]; + tensor input_173 = add(x = x_29, y = ret_out_9)[name = tensor("input_173")]; tensor xt_1_axes_0 = const()[name = tensor("xt_1_axes_0"), val = tensor([-1])]; - tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_2366, gamma = decoder_norm11_0_weight, x = input_333)[name = tensor("xt_1")]; - tensor var_2534 = const()[name = tensor("op_2534"), val = tensor([1, 12, 5, 256])]; - tensor var_2535 = reshape(shape = var_2534, x = xt_1)[name = tensor("op_2535")]; - tensor var_2536_perm_0 = const()[name = tensor("op_2536_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2539 = const()[name = tensor("op_2539"), val = tensor([5, 12, 256])]; - tensor var_2536 = transpose(perm = var_2536_perm_0, x = var_2535)[name = tensor("transpose_23")]; - tensor query_1 = reshape(shape = var_2539, x = var_2536)[name = tensor("query_1")]; + tensor xt_1 = layer_norm(axes = xt_1_axes_0, beta = decoder_norm11_0_bias, epsilon = var_1505, gamma = decoder_norm11_0_weight, x = input_173)[name = tensor("xt_1")]; + tensor var_1674 = const()[name = tensor("op_1674"), val = tensor([1, 12, 5, 256])]; + tensor var_1675 = reshape(shape = var_1674, x = xt_1)[name = tensor("op_1675")]; + tensor var_1676_perm_0 = const()[name = tensor("op_1676_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1679 = const()[name = tensor("op_1679"), val = tensor([5, 12, 256])]; + tensor var_1676 = transpose(perm = var_1676_perm_0, x = var_1675)[name = tensor("transpose_23")]; + tensor query_1 = reshape(shape = var_1679, x = var_1676)[name = tensor("query_1")]; tensor query_3_perm_0 = const()[name = tensor("query_3_perm_0"), val = tensor([1, 0, 2])]; tensor query_3 = transpose(perm = query_3_perm_0, x = query_1)[name = tensor("transpose_22")]; - tensor var_2562 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; - tensor concat_20 = const()[name = tensor("concat_20"), val = tensor([12, 5, 3, 256])]; - tensor var_2564 = reshape(shape = concat_20, x = var_2562)[name = tensor("op_2564")]; - tensor var_2565_axes_0 = const()[name = tensor("op_2565_axes_0"), val = tensor([0])]; - tensor var_2565 = expand_dims(axes = var_2565_axes_0, x = var_2564)[name = tensor("op_2565")]; - tensor var_2566_perm_0 = const()[name = tensor("op_2566_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2567_axes_0 = const()[name = tensor("op_2567_axes_0"), val = tensor([-2])]; - tensor var_2566 = transpose(perm = var_2566_perm_0, x = var_2565)[name = tensor("transpose_21")]; - tensor var_2567 = squeeze(axes = var_2567_axes_0, x = var_2566)[name = tensor("op_2567")]; + tensor var_1702 = linear(bias = decoder_self_attn2_0_in_proj_bias, weight = decoder_self_attn2_0_in_proj_weight, x = query_3)[name = tensor("linear_43")]; + tensor concat_21 = const()[name = tensor("concat_21"), val = tensor([12, 5, 3, 256])]; + tensor var_1704 = reshape(shape = concat_21, x = var_1702)[name = tensor("op_1704")]; + tensor var_1705_axes_0 = const()[name = tensor("op_1705_axes_0"), val = tensor([0])]; + tensor var_1705 = expand_dims(axes = var_1705_axes_0, x = var_1704)[name = tensor("op_1705")]; + tensor var_1706_perm_0 = const()[name = tensor("op_1706_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1707_axes_0 = const()[name = tensor("op_1707_axes_0"), val = tensor([-2])]; + tensor var_1706 = transpose(perm = var_1706_perm_0, x = var_1705)[name = tensor("transpose_21")]; + tensor var_1707 = squeeze(axes = var_1707_axes_0, x = var_1706)[name = tensor("op_1707")]; tensor q_11_begin_0 = const()[name = tensor("q_11_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_11_end_0 = const()[name = tensor("q_11_end_0"), val = tensor([1, 12, 5, 256])]; tensor q_11_end_mask_0 = const()[name = tensor("q_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_11_squeeze_mask_0 = const()[name = tensor("q_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_2567)[name = tensor("q_11")]; + tensor q_11 = slice_by_index(begin = q_11_begin_0, end = q_11_end_0, end_mask = q_11_end_mask_0, squeeze_mask = q_11_squeeze_mask_0, x = var_1707)[name = tensor("q_11")]; tensor k_11_begin_0 = const()[name = tensor("k_11_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_11_end_0 = const()[name = tensor("k_11_end_0"), val = tensor([2, 12, 5, 256])]; tensor k_11_end_mask_0 = const()[name = tensor("k_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_11_squeeze_mask_0 = const()[name = tensor("k_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_2567)[name = tensor("k_11")]; + tensor k_11 = slice_by_index(begin = k_11_begin_0, end = k_11_end_0, end_mask = k_11_end_mask_0, squeeze_mask = k_11_squeeze_mask_0, x = var_1707)[name = tensor("k_11")]; tensor v_11_begin_0 = const()[name = tensor("v_11_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_11_end_0 = const()[name = tensor("v_11_end_0"), val = tensor([3, 12, 5, 256])]; tensor v_11_end_mask_0 = const()[name = tensor("v_11_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_11_squeeze_mask_0 = const()[name = tensor("v_11_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_2567)[name = tensor("v_11")]; - tensor var_2575 = const()[name = tensor("op_2575"), val = tensor([12, 20, 64])]; - tensor var_2576 = reshape(shape = var_2575, x = q_11)[name = tensor("op_2576")]; + tensor v_11 = slice_by_index(begin = v_11_begin_0, end = v_11_end_0, end_mask = v_11_end_mask_0, squeeze_mask = v_11_squeeze_mask_0, x = var_1707)[name = tensor("v_11")]; + tensor var_1715 = const()[name = tensor("op_1715"), val = tensor([12, 20, 64])]; + tensor var_1716 = reshape(shape = var_1715, x = q_11)[name = tensor("op_1716")]; tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2582 = const()[name = tensor("op_2582"), val = tensor([12, 20, 64])]; - tensor var_2583 = reshape(shape = var_2582, x = k_11)[name = tensor("op_2583")]; + tensor var_1722 = const()[name = tensor("op_1722"), val = tensor([12, 20, 64])]; + tensor var_1723 = reshape(shape = var_1722, x = k_11)[name = tensor("op_1723")]; tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2589 = const()[name = tensor("op_2589"), val = tensor([12, 20, 64])]; - tensor var_2590 = reshape(shape = var_2589, x = v_11)[name = tensor("op_2590")]; + tensor var_1729 = const()[name = tensor("op_1729"), val = tensor([12, 20, 64])]; + tensor var_1730 = reshape(shape = var_1729, x = v_11)[name = tensor("op_1730")]; tensor v_13_perm_0 = const()[name = tensor("v_13_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2593 = const()[name = tensor("op_2593"), val = tensor([5, 4, 12, 64])]; - tensor q_13 = transpose(perm = q_13_perm_0, x = var_2576)[name = tensor("transpose_20")]; - tensor q_15 = reshape(shape = var_2593, x = q_13)[name = tensor("q_15")]; - tensor var_2595 = const()[name = tensor("op_2595"), val = tensor([5, 4, 12, 64])]; - tensor k_13 = transpose(perm = k_13_perm_0, x = var_2583)[name = tensor("transpose_19")]; - tensor k_15 = reshape(shape = var_2595, x = k_13)[name = tensor("k_15")]; - tensor var_2597 = const()[name = tensor("op_2597"), val = tensor([5, 4, 12, 64])]; - tensor v_13 = transpose(perm = v_13_perm_0, x = var_2590)[name = tensor("transpose_18")]; - tensor v_15 = reshape(shape = var_2597, x = v_13)[name = tensor("v_15")]; + tensor var_1733 = const()[name = tensor("op_1733"), val = tensor([5, 4, 12, 64])]; + tensor q_13 = transpose(perm = q_13_perm_0, x = var_1716)[name = tensor("transpose_20")]; + tensor q_15 = reshape(shape = var_1733, x = q_13)[name = tensor("q_15")]; + tensor var_1735 = const()[name = tensor("op_1735"), val = tensor([5, 4, 12, 64])]; + tensor k_13 = transpose(perm = k_13_perm_0, x = var_1723)[name = tensor("transpose_19")]; + tensor k_15 = reshape(shape = var_1735, x = k_13)[name = tensor("k_15")]; + tensor var_1737 = const()[name = tensor("op_1737"), val = tensor([5, 4, 12, 64])]; + tensor v_13 = transpose(perm = v_13_perm_0, x = var_1730)[name = tensor("transpose_18")]; + tensor v_15 = reshape(shape = var_1737, x = v_13)[name = tensor("v_15")]; tensor mul_1_y_0 = const()[name = tensor("mul_1_y_0"), val = tensor(0x1p-3)]; tensor mul_1 = mul(x = q_15, y = mul_1_y_0)[name = tensor("mul_1")]; tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; @@ -2281,30 +1666,30 @@ program(1.0) tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; tensor attn_output_1 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = softmax_0, y = v_15)[name = tensor("attn_output_1")]; - tensor var_2600 = const()[name = tensor("op_2600"), val = tensor([2, 0, 1, 3])]; - tensor var_2605 = const()[name = tensor("op_2605"), val = tensor([60, 256])]; - tensor var_2601 = transpose(perm = var_2600, x = attn_output_1)[name = tensor("transpose_17")]; - tensor attn_output_3 = reshape(shape = var_2605, x = var_2601)[name = tensor("attn_output_3")]; + tensor var_1740 = const()[name = tensor("op_1740"), val = tensor([2, 0, 1, 3])]; + tensor var_1745 = const()[name = tensor("op_1745"), val = tensor([60, 256])]; + tensor var_1741 = transpose(perm = var_1740, x = attn_output_1)[name = tensor("transpose_17")]; + tensor attn_output_3 = reshape(shape = var_1745, x = var_1741)[name = tensor("attn_output_3")]; tensor attn_output_5 = linear(bias = decoder_self_attn2_0_out_proj_bias, weight = decoder_self_attn2_0_out_proj_weight, x = attn_output_3)[name = tensor("linear_44")]; - tensor var_2609 = const()[name = tensor("op_2609"), val = tensor([12, 5, 256])]; - tensor attn_output_7 = reshape(shape = var_2609, x = attn_output_5)[name = tensor("attn_output_7")]; + tensor var_1749 = const()[name = tensor("op_1749"), val = tensor([12, 5, 256])]; + tensor attn_output_7 = reshape(shape = var_1749, x = attn_output_5)[name = tensor("attn_output_7")]; tensor sa2_out_1_perm_0 = const()[name = tensor("sa2_out_1_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out_1 = transpose(perm = sa2_out_1_perm_0, x = attn_output_7)[name = tensor("transpose_16")]; - tensor input_335 = add(x = query_1, y = sa2_out_1)[name = tensor("input_335")]; - tensor input_337_axes_0 = const()[name = tensor("input_337_axes_0"), val = tensor([-1])]; - tensor input_337 = layer_norm(axes = input_337_axes_0, beta = decoder_norm21_0_bias, epsilon = var_2366, gamma = decoder_norm21_0_weight, x = input_335)[name = tensor("input_337")]; - tensor input_339 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_337)[name = tensor("linear_45")]; - tensor input_341 = relu(x = input_339)[name = tensor("input_341")]; - tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_341)[name = tensor("linear_46")]; - tensor input_343 = add(x = input_337, y = ffn_out_1)[name = tensor("input_343")]; + tensor input_175 = add(x = query_1, y = sa2_out_1)[name = tensor("input_175")]; + tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; + tensor input_177 = layer_norm(axes = input_177_axes_0, beta = decoder_norm21_0_bias, epsilon = var_1505, gamma = decoder_norm21_0_weight, x = input_175)[name = tensor("input_177")]; + tensor input_179 = linear(bias = decoder_linear1_0_bias, weight = decoder_linear1_0_weight, x = input_177)[name = tensor("linear_45")]; + tensor input_181 = relu(x = input_179)[name = tensor("input_181")]; + tensor ffn_out_1 = linear(bias = decoder_linear2_0_bias, weight = decoder_linear2_0_weight, x = input_181)[name = tensor("linear_46")]; + tensor input_183 = add(x = input_177, y = ffn_out_1)[name = tensor("input_183")]; tensor xt_3_axes_0 = const()[name = tensor("xt_3_axes_0"), val = tensor([-1])]; - tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_2366, gamma = decoder_norm22_0_weight, x = input_343)[name = tensor("xt_3")]; - tensor var_2629 = const()[name = tensor("op_2629"), val = tensor([1, 5, 12, 256])]; - tensor x_63 = reshape(shape = var_2629, x = xt_3)[name = tensor("x_63")]; - tensor var_2631_perm_0 = const()[name = tensor("op_2631_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2635 = const()[name = tensor("op_2635"), val = tensor([12, 5, 256])]; - tensor var_2631 = transpose(perm = var_2631_perm_0, x = x_63)[name = tensor("transpose_15")]; - tensor x = reshape(shape = var_2635, x = var_2631)[name = tensor("x")]; + tensor xt_3 = layer_norm(axes = xt_3_axes_0, beta = decoder_norm22_0_bias, epsilon = var_1505, gamma = decoder_norm22_0_weight, x = input_183)[name = tensor("xt_3")]; + tensor var_1769 = const()[name = tensor("op_1769"), val = tensor([1, 5, 12, 256])]; + tensor x_31 = reshape(shape = var_1769, x = xt_3)[name = tensor("x_31")]; + tensor var_1771_perm_0 = const()[name = tensor("op_1771_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1775 = const()[name = tensor("op_1775"), val = tensor([12, 5, 256])]; + tensor var_1771 = transpose(perm = var_1771_perm_0, x = x_31)[name = tensor("transpose_15")]; + tensor x = reshape(shape = var_1775, x = var_1771)[name = tensor("x")]; tensor prev_kv_begin_0 = const()[name = tensor("prev_kv_begin_0"), val = tensor([1, 0, 0, 0, 0])]; tensor prev_kv_end_0 = const()[name = tensor("prev_kv_end_0"), val = tensor([2, 12, 4, 64, 64])]; tensor prev_kv_end_mask_0 = const()[name = tensor("prev_kv_end_mask_0"), val = tensor([false, true, true, true, true])]; @@ -2315,120 +1700,120 @@ program(1.0) tensor prev_scale_end_mask_0 = const()[name = tensor("prev_scale_end_mask_0"), val = tensor([false, true])]; tensor prev_scale_squeeze_mask_0 = const()[name = tensor("prev_scale_squeeze_mask_0"), val = tensor([true, false])]; tensor prev_scale = slice_by_index(begin = prev_scale_begin_0, end = prev_scale_end_0, end_mask = prev_scale_end_mask_0, squeeze_mask = prev_scale_squeeze_mask_0, x = dec_scale)[name = tensor("prev_scale")]; - tensor var_2643 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; - tensor var_2644 = const()[name = tensor("op_2644"), val = tensor([12, 5, 4, 64])]; - tensor var_2645 = reshape(shape = var_2644, x = var_2643)[name = tensor("op_2645")]; + tensor var_1783 = linear(bias = decoder_q_proj_1_bias, weight = decoder_q_proj_1_weight, x = x)[name = tensor("linear_47")]; + tensor var_1784 = const()[name = tensor("op_1784"), val = tensor([12, 5, 4, 64])]; + tensor var_1785 = reshape(shape = var_1784, x = var_1783)[name = tensor("op_1785")]; tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2649 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; - tensor var_2650 = const()[name = tensor("op_2650"), val = tensor(0x1p-3)]; - tensor var_2651 = mul(x = var_2649, y = var_2650)[name = tensor("op_2651")]; - tensor var_2652 = const()[name = tensor("op_2652"), val = tensor([12, 5, 4, 64])]; - tensor var_2653 = reshape(shape = var_2652, x = var_2651)[name = tensor("op_2653")]; + tensor var_1789 = linear(bias = decoder_k_proj_1_bias, weight = decoder_k_proj_1_weight, x = x)[name = tensor("linear_48")]; + tensor var_1790 = const()[name = tensor("op_1790"), val = tensor(0x1p-3)]; + tensor var_1791 = mul(x = var_1789, y = var_1790)[name = tensor("op_1791")]; + tensor var_1792 = const()[name = tensor("op_1792"), val = tensor([12, 5, 4, 64])]; + tensor var_1793 = reshape(shape = var_1792, x = var_1791)[name = tensor("op_1793")]; tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2657 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; - tensor var_2658 = const()[name = tensor("op_2658"), val = tensor([12, 5, 4, 64])]; - tensor var_2659 = reshape(shape = var_2658, x = var_2657)[name = tensor("op_2659")]; + tensor var_1797 = linear(bias = decoder_v_proj_1_bias, weight = decoder_v_proj_1_weight, x = x)[name = tensor("linear_49")]; + tensor var_1798 = const()[name = tensor("op_1798"), val = tensor([12, 5, 4, 64])]; + tensor var_1799 = reshape(shape = var_1798, x = var_1797)[name = tensor("op_1799")]; tensor v_17_perm_0 = const()[name = tensor("v_17_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor input_347 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; + tensor input_187 = linear(bias = decoder_g_proj_1_bias, weight = decoder_g_proj_1_weight, x = x)[name = tensor("linear_50")]; tensor sqrt_s0 = sqrt(x = prev_scale)[name = tensor("sqrt_s0")]; tensor s_t = add(x = prev_scale, y = cumsum_mask_1)[name = tensor("s_t")]; - tensor const_48 = const()[name = tensor("const_48"), val = tensor(0x1.fffffep+127)]; - tensor clip_43 = clip(alpha = var_2371, beta = const_48, x = s_t)[name = tensor("clip_43")]; + tensor const_32 = const()[name = tensor("const_32"), val = tensor(0x1.fffffep+127)]; + tensor clip_43 = clip(alpha = var_1510, beta = const_32, x = s_t)[name = tensor("clip_43")]; tensor sqrt_s_t = sqrt(x = clip_43)[name = tensor("sqrt_s_t")]; tensor qk_transpose_x_1 = const()[name = tensor("qk_transpose_x_1"), val = tensor(false)]; tensor qk_transpose_y_1 = const()[name = tensor("qk_transpose_y_1"), val = tensor(true)]; - tensor k_17 = transpose(perm = k_17_perm_0, x = var_2653)[name = tensor("transpose_13")]; - tensor q_17 = transpose(perm = q_17_perm_0, x = var_2645)[name = tensor("transpose_14")]; + tensor k_17 = transpose(perm = k_17_perm_0, x = var_1793)[name = tensor("transpose_13")]; + tensor q_17 = transpose(perm = q_17_perm_0, x = var_1785)[name = tensor("transpose_14")]; tensor qk = matmul(transpose_x = qk_transpose_x_1, transpose_y = qk_transpose_y_1, x = q_17, y = k_17)[name = tensor("qk")]; - tensor var_2674 = const()[name = tensor("op_2674"), val = tensor([5, 1])]; - tensor var_2675 = reshape(shape = var_2674, x = sqrt_s_t)[name = tensor("op_2675")]; - tensor M = real_div(x = causal_with_valid_1, y = var_2675)[name = tensor("M")]; - tensor var_2677 = mul(x = qk, y = M)[name = tensor("op_2677")]; + tensor var_1814 = const()[name = tensor("op_1814"), val = tensor([5, 1])]; + tensor var_1815 = reshape(shape = var_1814, x = sqrt_s_t)[name = tensor("op_1815")]; + tensor M = real_div(x = causal_with_valid_1, y = var_1815)[name = tensor("M")]; + tensor var_1817 = mul(x = qk, y = M)[name = tensor("op_1817")]; tensor inner_transpose_x_0 = const()[name = tensor("inner_transpose_x_0"), val = tensor(false)]; tensor inner_transpose_y_0 = const()[name = tensor("inner_transpose_y_0"), val = tensor(false)]; - tensor v_17 = transpose(perm = v_17_perm_0, x = var_2659)[name = tensor("transpose_12")]; - tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_2677, y = v_17)[name = tensor("inner")]; - tensor var_2679_transpose_x_0 = const()[name = tensor("op_2679_transpose_x_0"), val = tensor(false)]; - tensor var_2679_transpose_y_0 = const()[name = tensor("op_2679_transpose_y_0"), val = tensor(false)]; - tensor var_2679 = matmul(transpose_x = var_2679_transpose_x_0, transpose_y = var_2679_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_2679")]; - tensor var_2680 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_2680")]; - tensor var_2681 = const()[name = tensor("op_2681"), val = tensor([1, 1, 5, 1])]; - tensor var_2682 = reshape(shape = var_2681, x = var_2680)[name = tensor("op_2682")]; - tensor cross = mul(x = var_2679, y = var_2682)[name = tensor("cross")]; + tensor v_17 = transpose(perm = v_17_perm_0, x = var_1799)[name = tensor("transpose_12")]; + tensor inner = matmul(transpose_x = inner_transpose_x_0, transpose_y = inner_transpose_y_0, x = var_1817, y = v_17)[name = tensor("inner")]; + tensor var_1819_transpose_x_0 = const()[name = tensor("op_1819_transpose_x_0"), val = tensor(false)]; + tensor var_1819_transpose_y_0 = const()[name = tensor("op_1819_transpose_y_0"), val = tensor(false)]; + tensor var_1819 = matmul(transpose_x = var_1819_transpose_x_0, transpose_y = var_1819_transpose_y_0, x = q_17, y = prev_kv)[name = tensor("op_1819")]; + tensor var_1820 = real_div(x = sqrt_s0, y = sqrt_s_t)[name = tensor("op_1820")]; + tensor var_1821 = const()[name = tensor("op_1821"), val = tensor([1, 1, 5, 1])]; + tensor var_1822 = reshape(shape = var_1821, x = var_1820)[name = tensor("op_1822")]; + tensor cross = mul(x = var_1819, y = var_1822)[name = tensor("cross")]; tensor out_31 = add(x = inner, y = cross)[name = tensor("out_31")]; - tensor v_masked = mul(x = v_17, y = var_2505)[name = tensor("v_masked")]; - tensor var_2688 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_2688")]; - tensor var_2690_transpose_x_1 = const()[name = tensor("op_2690_transpose_x_1"), val = tensor(true)]; - tensor var_2690_transpose_y_1 = const()[name = tensor("op_2690_transpose_y_1"), val = tensor(false)]; - tensor var_2690 = matmul(transpose_x = var_2690_transpose_x_1, transpose_y = var_2690_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_2690")]; - tensor new_kv_unnorm = add(x = var_2688, y = var_2690)[name = tensor("new_kv_unnorm")]; - tensor new_scale = add(x = prev_scale, y = var_2513)[name = tensor("new_scale")]; - tensor const_49 = const()[name = tensor("const_49"), val = tensor(0x1.fffffep+127)]; - tensor clip_44 = clip(alpha = var_2371, beta = const_49, x = new_scale)[name = tensor("clip_44")]; + tensor v_masked = mul(x = v_17, y = var_1645)[name = tensor("v_masked")]; + tensor var_1828 = mul(x = prev_kv, y = sqrt_s0)[name = tensor("op_1828")]; + tensor var_1830_transpose_x_1 = const()[name = tensor("op_1830_transpose_x_1"), val = tensor(true)]; + tensor var_1830_transpose_y_1 = const()[name = tensor("op_1830_transpose_y_1"), val = tensor(false)]; + tensor var_1830 = matmul(transpose_x = var_1830_transpose_x_1, transpose_y = var_1830_transpose_y_1, x = k_17, y = v_masked)[name = tensor("op_1830")]; + tensor new_kv_unnorm = add(x = var_1828, y = var_1830)[name = tensor("new_kv_unnorm")]; + tensor new_scale = add(x = prev_scale, y = var_1653)[name = tensor("new_scale")]; + tensor const_33 = const()[name = tensor("const_33"), val = tensor(0x1.fffffep+127)]; + tensor clip_44 = clip(alpha = var_1510, beta = const_33, x = new_scale)[name = tensor("clip_44")]; tensor sqrt_new_scale = sqrt(x = clip_44)[name = tensor("sqrt_new_scale")]; tensor nkv = real_div(x = new_kv_unnorm, y = sqrt_new_scale)[name = tensor("nkv")]; - tensor var_2699_perm_0 = const()[name = tensor("op_2699_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1839_perm_0 = const()[name = tensor("op_1839_perm_0"), val = tensor([0, 2, 1, 3])]; tensor out_33_axes_0 = const()[name = tensor("out_33_axes_0"), val = tensor([-1])]; - tensor var_2699 = transpose(perm = var_2699_perm_0, x = out_31)[name = tensor("transpose_11")]; - tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_2368, x = var_2699)[name = tensor("out_33")]; - tensor var_2703 = const()[name = tensor("op_2703"), val = tensor([12, 5, 256])]; - tensor out = reshape(shape = var_2703, x = out_33)[name = tensor("out")]; - tensor var_2705 = silu(x = input_347)[name = tensor("op_2705")]; - tensor input_349 = mul(x = var_2705, y = out)[name = tensor("input_349")]; - tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_349)[name = tensor("linear_51")]; - tensor input_351 = add(x = x, y = ret_out)[name = tensor("input_351")]; + tensor var_1839 = transpose(perm = var_1839_perm_0, x = out_31)[name = tensor("transpose_11")]; + tensor out_33 = layer_norm(axes = out_33_axes_0, epsilon = var_1507, x = var_1839)[name = tensor("out_33")]; + tensor var_1843 = const()[name = tensor("op_1843"), val = tensor([12, 5, 256])]; + tensor out = reshape(shape = var_1843, x = out_33)[name = tensor("out")]; + tensor var_1845 = silu(x = input_187)[name = tensor("op_1845")]; + tensor input_189 = mul(x = var_1845, y = out)[name = tensor("input_189")]; + tensor ret_out = linear(bias = decoder_out_proj_1_bias, weight = decoder_out_proj_1_weight, x = input_189)[name = tensor("linear_51")]; + tensor input_191 = add(x = x, y = ret_out)[name = tensor("input_191")]; tensor xt_5_axes_0 = const()[name = tensor("xt_5_axes_0"), val = tensor([-1])]; - tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_2366, gamma = decoder_norm11_1_weight, x = input_351)[name = tensor("xt_5")]; - tensor var_2715 = const()[name = tensor("op_2715"), val = tensor([1, 12, 5, 256])]; - tensor var_2716 = reshape(shape = var_2715, x = xt_5)[name = tensor("op_2716")]; - tensor var_2717_perm_0 = const()[name = tensor("op_2717_perm_0"), val = tensor([0, 2, 1, 3])]; - tensor var_2720 = const()[name = tensor("op_2720"), val = tensor([5, 12, 256])]; - tensor var_2717 = transpose(perm = var_2717_perm_0, x = var_2716)[name = tensor("transpose_10")]; - tensor query_5 = reshape(shape = var_2720, x = var_2717)[name = tensor("query_5")]; + tensor xt_5 = layer_norm(axes = xt_5_axes_0, beta = decoder_norm11_1_bias, epsilon = var_1505, gamma = decoder_norm11_1_weight, x = input_191)[name = tensor("xt_5")]; + tensor var_1855 = const()[name = tensor("op_1855"), val = tensor([1, 12, 5, 256])]; + tensor var_1856 = reshape(shape = var_1855, x = xt_5)[name = tensor("op_1856")]; + tensor var_1857_perm_0 = const()[name = tensor("op_1857_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1860 = const()[name = tensor("op_1860"), val = tensor([5, 12, 256])]; + tensor var_1857 = transpose(perm = var_1857_perm_0, x = var_1856)[name = tensor("transpose_10")]; + tensor query_5 = reshape(shape = var_1860, x = var_1857)[name = tensor("query_5")]; tensor query_perm_0 = const()[name = tensor("query_perm_0"), val = tensor([1, 0, 2])]; tensor query = transpose(perm = query_perm_0, x = query_5)[name = tensor("transpose_9")]; - tensor var_2743 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; - tensor concat_21 = const()[name = tensor("concat_21"), val = tensor([12, 5, 3, 256])]; - tensor var_2745 = reshape(shape = concat_21, x = var_2743)[name = tensor("op_2745")]; - tensor var_2746_axes_0 = const()[name = tensor("op_2746_axes_0"), val = tensor([0])]; - tensor var_2746 = expand_dims(axes = var_2746_axes_0, x = var_2745)[name = tensor("op_2746")]; - tensor var_2747_perm_0 = const()[name = tensor("op_2747_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; - tensor var_2748_axes_0 = const()[name = tensor("op_2748_axes_0"), val = tensor([-2])]; - tensor var_2747 = transpose(perm = var_2747_perm_0, x = var_2746)[name = tensor("transpose_8")]; - tensor var_2748 = squeeze(axes = var_2748_axes_0, x = var_2747)[name = tensor("op_2748")]; + tensor var_1883 = linear(bias = decoder_self_attn2_1_in_proj_bias, weight = decoder_self_attn2_1_in_proj_weight, x = query)[name = tensor("linear_52")]; + tensor concat_22 = const()[name = tensor("concat_22"), val = tensor([12, 5, 3, 256])]; + tensor var_1885 = reshape(shape = concat_22, x = var_1883)[name = tensor("op_1885")]; + tensor var_1886_axes_0 = const()[name = tensor("op_1886_axes_0"), val = tensor([0])]; + tensor var_1886 = expand_dims(axes = var_1886_axes_0, x = var_1885)[name = tensor("op_1886")]; + tensor var_1887_perm_0 = const()[name = tensor("op_1887_perm_0"), val = tensor([-2, 1, 2, 0, 4])]; + tensor var_1888_axes_0 = const()[name = tensor("op_1888_axes_0"), val = tensor([-2])]; + tensor var_1887 = transpose(perm = var_1887_perm_0, x = var_1886)[name = tensor("transpose_8")]; + tensor var_1888 = squeeze(axes = var_1888_axes_0, x = var_1887)[name = tensor("op_1888")]; tensor q_19_begin_0 = const()[name = tensor("q_19_begin_0"), val = tensor([0, 0, 0, 0])]; tensor q_19_end_0 = const()[name = tensor("q_19_end_0"), val = tensor([1, 12, 5, 256])]; tensor q_19_end_mask_0 = const()[name = tensor("q_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor q_19_squeeze_mask_0 = const()[name = tensor("q_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_2748)[name = tensor("q_19")]; + tensor q_19 = slice_by_index(begin = q_19_begin_0, end = q_19_end_0, end_mask = q_19_end_mask_0, squeeze_mask = q_19_squeeze_mask_0, x = var_1888)[name = tensor("q_19")]; tensor k_19_begin_0 = const()[name = tensor("k_19_begin_0"), val = tensor([1, 0, 0, 0])]; tensor k_19_end_0 = const()[name = tensor("k_19_end_0"), val = tensor([2, 12, 5, 256])]; tensor k_19_end_mask_0 = const()[name = tensor("k_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor k_19_squeeze_mask_0 = const()[name = tensor("k_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_2748)[name = tensor("k_19")]; + tensor k_19 = slice_by_index(begin = k_19_begin_0, end = k_19_end_0, end_mask = k_19_end_mask_0, squeeze_mask = k_19_squeeze_mask_0, x = var_1888)[name = tensor("k_19")]; tensor v_19_begin_0 = const()[name = tensor("v_19_begin_0"), val = tensor([2, 0, 0, 0])]; tensor v_19_end_0 = const()[name = tensor("v_19_end_0"), val = tensor([3, 12, 5, 256])]; tensor v_19_end_mask_0 = const()[name = tensor("v_19_end_mask_0"), val = tensor([false, true, true, true])]; tensor v_19_squeeze_mask_0 = const()[name = tensor("v_19_squeeze_mask_0"), val = tensor([true, false, false, false])]; - tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_2748)[name = tensor("v_19")]; - tensor var_2756 = const()[name = tensor("op_2756"), val = tensor([12, 20, 64])]; - tensor var_2757 = reshape(shape = var_2756, x = q_19)[name = tensor("op_2757")]; + tensor v_19 = slice_by_index(begin = v_19_begin_0, end = v_19_end_0, end_mask = v_19_end_mask_0, squeeze_mask = v_19_squeeze_mask_0, x = var_1888)[name = tensor("v_19")]; + tensor var_1896 = const()[name = tensor("op_1896"), val = tensor([12, 20, 64])]; + tensor var_1897 = reshape(shape = var_1896, x = q_19)[name = tensor("op_1897")]; tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2763 = const()[name = tensor("op_2763"), val = tensor([12, 20, 64])]; - tensor var_2764 = reshape(shape = var_2763, x = k_19)[name = tensor("op_2764")]; + tensor var_1903 = const()[name = tensor("op_1903"), val = tensor([12, 20, 64])]; + tensor var_1904 = reshape(shape = var_1903, x = k_19)[name = tensor("op_1904")]; tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2770 = const()[name = tensor("op_2770"), val = tensor([12, 20, 64])]; - tensor var_2771 = reshape(shape = var_2770, x = v_19)[name = tensor("op_2771")]; + tensor var_1910 = const()[name = tensor("op_1910"), val = tensor([12, 20, 64])]; + tensor var_1911 = reshape(shape = var_1910, x = v_19)[name = tensor("op_1911")]; tensor v_21_perm_0 = const()[name = tensor("v_21_perm_0"), val = tensor([1, 0, 2])]; - tensor var_2774 = const()[name = tensor("op_2774"), val = tensor([5, 4, 12, 64])]; - tensor q_21 = transpose(perm = q_21_perm_0, x = var_2757)[name = tensor("transpose_7")]; - tensor q = reshape(shape = var_2774, x = q_21)[name = tensor("q")]; - tensor var_2776 = const()[name = tensor("op_2776"), val = tensor([5, 4, 12, 64])]; - tensor k_21 = transpose(perm = k_21_perm_0, x = var_2764)[name = tensor("transpose_6")]; - tensor k = reshape(shape = var_2776, x = k_21)[name = tensor("k")]; - tensor var_2778 = const()[name = tensor("op_2778"), val = tensor([5, 4, 12, 64])]; - tensor v_21 = transpose(perm = v_21_perm_0, x = var_2771)[name = tensor("transpose_5")]; - tensor v = reshape(shape = var_2778, x = v_21)[name = tensor("v")]; + tensor var_1914 = const()[name = tensor("op_1914"), val = tensor([5, 4, 12, 64])]; + tensor q_21 = transpose(perm = q_21_perm_0, x = var_1897)[name = tensor("transpose_7")]; + tensor q = reshape(shape = var_1914, x = q_21)[name = tensor("q")]; + tensor var_1916 = const()[name = tensor("op_1916"), val = tensor([5, 4, 12, 64])]; + tensor k_21 = transpose(perm = k_21_perm_0, x = var_1904)[name = tensor("transpose_6")]; + tensor k = reshape(shape = var_1916, x = k_21)[name = tensor("k")]; + tensor var_1918 = const()[name = tensor("op_1918"), val = tensor([5, 4, 12, 64])]; + tensor v_21 = transpose(perm = v_21_perm_0, x = var_1911)[name = tensor("transpose_5")]; + tensor v = reshape(shape = var_1918, x = v_21)[name = tensor("v")]; tensor mul_3_y_0 = const()[name = tensor("mul_3_y_0"), val = tensor(0x1p-3)]; tensor mul_3 = mul(x = q, y = mul_3_y_0)[name = tensor("mul_3")]; tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; @@ -2439,50 +1824,50 @@ program(1.0) tensor attn_output_9_transpose_x_0 = const()[name = tensor("attn_output_9_transpose_x_0"), val = tensor(false)]; tensor attn_output_9_transpose_y_0 = const()[name = tensor("attn_output_9_transpose_y_0"), val = tensor(false)]; tensor attn_output_9 = matmul(transpose_x = attn_output_9_transpose_x_0, transpose_y = attn_output_9_transpose_y_0, x = softmax_1, y = v)[name = tensor("attn_output_9")]; - tensor var_2781 = const()[name = tensor("op_2781"), val = tensor([2, 0, 1, 3])]; - tensor var_2786 = const()[name = tensor("op_2786"), val = tensor([60, 256])]; - tensor var_2782 = transpose(perm = var_2781, x = attn_output_9)[name = tensor("transpose_4")]; - tensor attn_output_11 = reshape(shape = var_2786, x = var_2782)[name = tensor("attn_output_11")]; + tensor var_1921 = const()[name = tensor("op_1921"), val = tensor([2, 0, 1, 3])]; + tensor var_1926 = const()[name = tensor("op_1926"), val = tensor([60, 256])]; + tensor var_1922 = transpose(perm = var_1921, x = attn_output_9)[name = tensor("transpose_4")]; + tensor attn_output_11 = reshape(shape = var_1926, x = var_1922)[name = tensor("attn_output_11")]; tensor attn_output_13 = linear(bias = decoder_self_attn2_1_out_proj_bias, weight = decoder_self_attn2_1_out_proj_weight, x = attn_output_11)[name = tensor("linear_53")]; - tensor var_2790 = const()[name = tensor("op_2790"), val = tensor([12, 5, 256])]; - tensor attn_output = reshape(shape = var_2790, x = attn_output_13)[name = tensor("attn_output")]; + tensor var_1930 = const()[name = tensor("op_1930"), val = tensor([12, 5, 256])]; + tensor attn_output = reshape(shape = var_1930, x = attn_output_13)[name = tensor("attn_output")]; tensor sa2_out_perm_0 = const()[name = tensor("sa2_out_perm_0"), val = tensor([1, 0, 2])]; tensor sa2_out = transpose(perm = sa2_out_perm_0, x = attn_output)[name = tensor("transpose_3")]; - tensor input_353 = add(x = query_5, y = sa2_out)[name = tensor("input_353")]; - tensor input_355_axes_0 = const()[name = tensor("input_355_axes_0"), val = tensor([-1])]; - tensor input_355 = layer_norm(axes = input_355_axes_0, beta = decoder_norm21_1_bias, epsilon = var_2366, gamma = decoder_norm21_1_weight, x = input_353)[name = tensor("input_355")]; - tensor input_357 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_355)[name = tensor("linear_54")]; - tensor input_359 = relu(x = input_357)[name = tensor("input_359")]; - tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_359)[name = tensor("linear_55")]; - tensor input_361 = add(x = input_355, y = ffn_out)[name = tensor("input_361")]; + tensor input_193 = add(x = query_5, y = sa2_out)[name = tensor("input_193")]; + tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; + tensor input_195 = layer_norm(axes = input_195_axes_0, beta = decoder_norm21_1_bias, epsilon = var_1505, gamma = decoder_norm21_1_weight, x = input_193)[name = tensor("input_195")]; + tensor input_197 = linear(bias = decoder_linear1_1_bias, weight = decoder_linear1_1_weight, x = input_195)[name = tensor("linear_54")]; + tensor input_199 = relu(x = input_197)[name = tensor("input_199")]; + tensor ffn_out = linear(bias = decoder_linear2_1_bias, weight = decoder_linear2_1_weight, x = input_199)[name = tensor("linear_55")]; + tensor input_201 = add(x = input_195, y = ffn_out)[name = tensor("input_201")]; tensor xt_axes_0 = const()[name = tensor("xt_axes_0"), val = tensor([-1])]; - tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_2366, gamma = decoder_norm22_1_weight, x = input_361)[name = tensor("xt")]; - tensor var_2810 = const()[name = tensor("op_2810"), val = tensor([1, 5, 12, 256])]; - tensor input = reshape(shape = var_2810, x = xt)[name = tensor("input")]; - tensor var_2812 = const()[name = tensor("op_2812"), val = tensor([-1])]; - tensor var_2813 = reduce_l2_norm(axes = var_2812, keep_dims = var_2369, x = input)[name = tensor("op_2813")]; - tensor const_58 = const()[name = tensor("const_58"), val = tensor(0x1.fffffep+127)]; - tensor clip_45 = clip(alpha = var_2360, beta = const_58, x = var_2813)[name = tensor("clip_45")]; - tensor var_2815 = real_div(x = input, y = clip_45)[name = tensor("op_2815")]; - tensor concat_25 = const()[name = tensor("concat_25"), val = tensor([5, 1, 256])]; - tensor reshape_0 = reshape(shape = concat_25, x = emb)[name = tensor("reshape_0")]; + tensor xt = layer_norm(axes = xt_axes_0, beta = decoder_norm22_1_bias, epsilon = var_1505, gamma = decoder_norm22_1_weight, x = input_201)[name = tensor("xt")]; + tensor var_1950 = const()[name = tensor("op_1950"), val = tensor([1, 5, 12, 256])]; + tensor input = reshape(shape = var_1950, x = xt)[name = tensor("input")]; + tensor var_1952 = const()[name = tensor("op_1952"), val = tensor([-1])]; + tensor var_1953 = reduce_l2_norm(axes = var_1952, keep_dims = var_1508, x = input)[name = tensor("op_1953")]; + tensor const_42 = const()[name = tensor("const_42"), val = tensor(0x1.fffffep+127)]; + tensor clip_45 = clip(alpha = var_1500, beta = const_42, x = var_1953)[name = tensor("clip_45")]; + tensor var_1955 = real_div(x = input, y = clip_45)[name = tensor("op_1955")]; + tensor concat_26 = const()[name = tensor("concat_26"), val = tensor([5, 1, 256])]; + tensor reshape_0 = reshape(shape = concat_26, x = emb)[name = tensor("reshape_0")]; tensor transpose_1_perm_0 = const()[name = tensor("transpose_1_perm_0"), val = tensor([0, 1, 3, 2])]; - tensor concat_26 = const()[name = tensor("concat_26"), val = tensor([5, 256, 12])]; - tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_2815)[name = tensor("transpose_2")]; - tensor reshape_1 = reshape(shape = concat_26, x = transpose_1)[name = tensor("reshape_1")]; + tensor concat_27 = const()[name = tensor("concat_27"), val = tensor([5, 256, 12])]; + tensor transpose_1 = transpose(perm = transpose_1_perm_0, x = var_1955)[name = tensor("transpose_2")]; + tensor reshape_1 = reshape(shape = concat_27, x = transpose_1)[name = tensor("reshape_1")]; tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(false)]; tensor matmul_2 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = reshape_0, y = reshape_1)[name = tensor("matmul_2")]; - tensor concat_29 = const()[name = tensor("concat_29"), val = tensor([1, 5, 12])]; - tensor reshape_2 = reshape(shape = concat_29, x = matmul_2)[name = tensor("reshape_2")]; + tensor concat_30 = const()[name = tensor("concat_30"), val = tensor([1, 5, 12])]; + tensor reshape_2 = reshape(shape = concat_30, x = matmul_2)[name = tensor("reshape_2")]; tensor output_begin_0 = const()[name = tensor("output_begin_0"), val = tensor([0, 0, 1])]; tensor output_end_0 = const()[name = tensor("output_end_0"), val = tensor([1, 5, 11])]; tensor output_end_mask_0 = const()[name = tensor("output_end_mask_0"), val = tensor([true, true, false])]; tensor output = slice_by_index(begin = output_begin_0, end = output_end_0, end_mask = output_end_mask_0, x = reshape_2)[name = tensor("output")]; - tensor probs = sigmoid(x = output)[name = tensor("op_2819")]; - tensor var_2821_axis_0 = const()[name = tensor("op_2821_axis_0"), val = tensor(0)]; - tensor dec_kv_new = stack(axis = var_2821_axis_0, values = (var_2517, nkv))[name = tensor("op_2821")]; - tensor var_2823_axis_0 = const()[name = tensor("op_2823_axis_0"), val = tensor(0)]; - tensor dec_scale_new = stack(axis = var_2823_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_2823")]; + tensor probs = sigmoid(x = output)[name = tensor("op_1959")]; + tensor var_1961_axis_0 = const()[name = tensor("op_1961_axis_0"), val = tensor(0)]; + tensor dec_kv_new = stack(axis = var_1961_axis_0, values = (var_1657, nkv))[name = tensor("op_1961")]; + tensor var_1963_axis_0 = const()[name = tensor("op_1963_axis_0"), val = tensor(0)]; + tensor dec_scale_new = stack(axis = var_1963_axis_0, values = (new_scale_9, new_scale))[name = tensor("op_1963")]; } -> (probs, enc_kv_new, enc_scale_new, enc_conv_window_new, enc_conv_filled_new, cnn_window_new, dec_kv_new, dec_scale_new); } \ No newline at end of file diff --git a/optimized/DIHARD III/ls_eend_dih3_500ms.mlpackage/Data/com.apple.CoreML/model.mlmodel b/optimized/DIHARD III/ls_eend_dih3_500ms.mlpackage/Data/com.apple.CoreML/model.mlmodel index f92eebde82e793913e7deb238fd9bcd924f13009..1c88258d108f7f79fe473d1badaced85b777a7aa 100644 --- a/optimized/DIHARD III/ls_eend_dih3_500ms.mlpackage/Data/com.apple.CoreML/model.mlmodel +++ b/optimized/DIHARD III/ls_eend_dih3_500ms.mlpackage/Data/com.apple.CoreML/model.mlmodel @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0c07a85fd1ea15da9c55b8c4ad28ea6e6cfc4f165d2046696d4cf62031ec5edf -size 352371 +oid sha256:5f6b50cf73edd756afd26269fccfd35e845cfc6ca7d12601818a41f5e06cbffb +size 255681 diff --git a/optimized/DIHARD III/ls_eend_dih3_500ms.mlpackage/Manifest.json b/optimized/DIHARD III/ls_eend_dih3_500ms.mlpackage/Manifest.json index 4848f28ab10f7582236ed839b9f20ec9db4bbb5f..5001d58dd7fce5aae7abde48318cfd3978ac00b3 100644 --- a/optimized/DIHARD III/ls_eend_dih3_500ms.mlpackage/Manifest.json +++ b/optimized/DIHARD III/ls_eend_dih3_500ms.mlpackage/Manifest.json @@ -1,18 +1,18 @@ { "fileFormatVersion": "1.0.0", "itemInfoEntries": { - "0109C4B7-E7D9-426E-B27A-A8509817CDDE": { - "author": "com.apple.CoreML", - "description": "CoreML Model Specification", - "name": "model.mlmodel", - "path": "com.apple.CoreML/model.mlmodel" - }, - "2B4218E7-CB24-4742-959B-0477910E29BF": { + "2EA3F143-91F5-46E2-A753-82A9B7C76237": { "author": "com.apple.CoreML", "description": "CoreML Model Weights", "name": "weights", "path": "com.apple.CoreML/weights" + }, + "5FDC74D5-1E8C-4085-B592-689D735CF0ED": { + "author": "com.apple.CoreML", + "description": "CoreML Model Specification", + "name": "model.mlmodel", + "path": "com.apple.CoreML/model.mlmodel" } }, - "rootModelIdentifier": "0109C4B7-E7D9-426E-B27A-A8509817CDDE" + "rootModelIdentifier": "5FDC74D5-1E8C-4085-B592-689D735CF0ED" }